fix(cr-01/02/03/08/09/10/11/13): 会话世代、损耗共享基线、任务活性与工作台包

- CR-01:users.session_epoch(V172)+ JWT sessionEpoch;登录/登出原子推进,
  check_login/requireUser 先比世代再做设备比对;登出不再依赖 machine CAS
- CR-02/09:cost-guard 基线落 Redis(CostGuardBaselineStore),按 elapsed 归一化;
  非持锁实例刷新共享密钥快照;pause() 返回写成功,失败不通知且不推进基线
- CR-03:stuck_requeue_count(V173,上限 20)与 retry_count 分账;终态父任务心跳
  停止 touchRunning 并 markFailedPermanent;良性停续期 markLost=false
- CR-08:工作台包对象 key 加 UUID、0 字节拒绝、同 objectKey 幂等、同版本重复拒绝、
  前端 confirm 超时抬到 10 分钟
- CR-10:Excel onSheetTotal 统一在 onHeader 之后回调(xlsx/xls)
- CR-13/11:迁移头/文件名一致断言(V170 唯一豁免);V167 前置由 CompanyService 同事务
  复制菜单树,已应用迁移未回改
This commit is contained in:
2026-10-25 03:40:15 +08:00
parent 112c61afa8
commit c2f429f112
43 changed files with 1003 additions and 95 deletions
@@ -61,8 +61,11 @@ export async function confirmWorkbenchClientPackage(
fileName: string,
version: string,
): Promise<WorkbenchClientPackageItem | null> {
// confirm 要服务端流式复算最多 512MB 的 sha256;默认 30s 会在慢 MinIO/跨地域回源时
// 先超时,而服务端可能已经落库——管理员重试会得到「已登记」的错位(CR-08)。
const { data } = await http.post<unknown>(WORKBENCH_CLIENT_CONFIRM_ENDPOINT, null, {
params: { object_key: objectKey, file_name: fileName, version },
timeout: 600_000,
})
return parseWorkbenchClientPackageUpload(data)
}
@@ -94,5 +97,11 @@ export async function uploadWorkbenchClientPackage(
onProgress(total > 0 ? Math.min(Math.round((event.loaded / total) * 100), 100) : 0)
},
})
return confirmWorkbenchClientPackage(target.objectKey, target.fileName, target.version)
try {
return await confirmWorkbenchClientPackage(target.objectKey, target.fileName, target.version)
} catch (error) {
// 对象已经 PUT 成功,confirm 失败不等于上传失败;引导管理员先刷新列表,避免重复上传
const reason = error instanceof Error ? error.message : String(error)
throw new Error(`安装包已上传,但确认未完成(${reason})。请先刷新列表确认,不要立即重复上传。`)
}
}
@@ -126,16 +126,13 @@ public final class ExcelStreamReader {
SheetCollector collector = new SheetCollector(sheetName, sheetNo, handler, recorder);
XSSFSheetXMLHandler sheetHandler =
new XSSFSheetXMLHandler(styles, sharedStrings, collector, formatter, false);
DimensionFilter filter = new DimensionFilter(parser, maxRows -> {
try {
handler.onSheetTotal(sheetName, sheetNo, maxRows);
} catch (Exception ex) {
throw recorder.record(ex);
}
});
// dimension 在 sheetData 之前出现,先缓存;等 onHeader 回调之后再对外发 onSheetTotal,
// 保持旧 EasyExcel「先表头、后总行数」的顺序(CR-10)。
DimensionFilter filter = new DimensionFilter(parser, collector::setApproximateTotalRows);
filter.setContentHandler(sheetHandler);
parser.setContentHandler(filter);
parser.parse(new InputSource(sheet));
collector.reportTotalIfNeeded();
}
private static void readHssf(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
@@ -152,11 +149,12 @@ public final class ExcelStreamReader {
final String sheetName = sheet.getSheetName();
final int currentSheetNo = sheetNo;
final int lastRow = sheet.getLastRowNum();
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
final Map<Integer, String> header = rowToMap(sheet.getRow(0), formatter);
if (!header.isEmpty()) {
callRecorder(recorder, () -> handler.onHeader(sheetName, currentSheetNo, header));
}
// 旧顺序是「表头 → 总行数」;xls 路径此前反了,统一回来(CR-10)
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
for (int rowIndex = 1; rowIndex <= lastRow; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row == null) {
@@ -274,6 +272,9 @@ public final class ExcelStreamReader {
private final FailureRecorder recorder;
private Map<Integer, String> header = Map.of();
private Map<Integer, String> row = new LinkedHashMap<>();
private Integer approximateTotalRows;
private boolean headerReported;
private boolean totalReported;
private SheetCollector(String sheetName, int sheetNo, SheetRowHandler handler, FailureRecorder recorder) {
this.sheetName = sheetName;
@@ -282,6 +283,21 @@ public final class ExcelStreamReader {
this.recorder = recorder;
}
/** dimension 过滤器的回调:只缓存,等 onHeader 之后才对外发。 */
private void setApproximateTotalRows(Integer rows) {
this.approximateTotalRows = rows;
reportTotalIfNeeded();
}
private void reportTotalIfNeeded() {
if (!headerReported || totalReported) {
return;
}
totalReported = true;
callRecorder(recorder,
() -> handler.onSheetTotal(sheetName, sheetNo, approximateTotalRows));
}
@Override
public void startRow(int rowNum) {
row = new LinkedHashMap<>();
@@ -294,7 +310,9 @@ public final class ExcelStreamReader {
}
if (rowNum == 0) {
header = new LinkedHashMap<>(row);
headerReported = true;
callRecorder(recorder, () -> handler.onHeader(sheetName, sheetNo, header));
reportTotalIfNeeded();
} else {
Map<Integer, String> values = new LinkedHashMap<>(row);
int rowIndex = rowNum;
@@ -49,4 +49,10 @@ public class AdminUserEntity {
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
private LocalDateTime updatedAt;
private String machine;
/**
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
* 与 {@code LoginUserEntity.sessionEpoch} 同列,改一处必须同改两处。
*/
@TableField("session_epoch")
private Long sessionEpoch;
}
@@ -72,6 +72,9 @@ public class AdminAuthSupport {
// 吊销先于单设备比对:那条路径第一行就是 if (exempt) return;(超管豁免),
// 把吊销挂在它下面等于超管永不见吊销
DeviceSessionPolicy.assertNotRevoked(user.getMachine(), user.getId(), user.getUsername());
// 世代校验同样先于单设备比对,且不受 singleDevice 开关支配:登出/新登录后旧令牌一律失效
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
// 单设备登录:被新设备顶下线的旧 token 在此统一拦截(全站 requireUser 调用点自动生效)
if (authProperties.isSingleDeviceEnabled()) {
DeviceSessionPolicy.assertSameDevice(user.getMachine(), DeviceSessionPolicy.claimDeviceId(claims),
@@ -91,6 +91,30 @@ public final class DeviceSessionPolicy {
return raw == null ? "" : raw.toString().trim();
}
/**
* 从 JWT claims 中提取会话世代;缺失(本次升级前签发的旧令牌)按 0 读取,
* 与 {@code users.session_epoch} 的列默认值一致,升级不会统一踢掉存量会话。
*/
public static long claimSessionEpoch(Claims claims) {
Object raw = claims == null ? null : claims.get("sessionEpoch");
// JwtService 签发的是 Number;非 Number 一律视同缺失(按 0),不在这里做字符串兼容,
// 避免为一种不会出现的 claim 形状新增 catch(CatchSilenceRatchet 满格)。
return raw instanceof Number number ? number.longValue() : 0L;
}
/**
* 会话世代必须与库中值相等。登出与每次新登录都会 +1,因此登出前签发的令牌在
* 下一次请求即失效;不依赖 {@code machine} 上的 revoked@ 标记,也不受单设备开关支配。
*/
public static void assertSessionEpoch(Long storedEpoch, long claimEpoch, Long userId, String username) {
long current = storedEpoch == null ? 0L : storedEpoch;
if (current != claimEpoch) {
log.warn("[auth] 会话世代失效被拒 userId={} username={} stored={} claim={}",
userId, username, current, claimEpoch);
throw new BusinessException(401, "登录已过期,请重新登录");
}
}
/**
* 校验请求携带的 token 是否仍属于账号当前绑定的设备。
* 超管豁免;machine 为空(尚未绑定)放行;不匹配抛 4011。
@@ -85,13 +85,19 @@ public class JwtService {
return key;
}
public String issue(Long userId, String username, String deviceId) {
/**
* 签发令牌。{@code sessionEpoch} 必须取自 users.session_epoch(登录/登出各推进一次),
* 鉴权时与库中值比对;旧令牌没有这个 claim 时按 0 读取,见
* {@link DeviceSessionPolicy#claimSessionEpoch(Claims)}。
*/
public String issue(Long userId, String username, String deviceId, long sessionEpoch) {
Instant now = Instant.now();
Instant exp = now.plus(Duration.ofHours(props.getJwtTtlHours()));
return Jwts.builder()
.subject(String.valueOf(userId))
.claim("username", username)
.claim("deviceId", deviceId)
.claim("sessionEpoch", sessionEpoch)
.issuedAt(Date.from(now))
.expiration(Date.from(exp))
.signWith(signingKey())
@@ -74,6 +74,22 @@ public final class NodeSharedStore<K, V> {
return remote;
}
/**
* 远端优先;远端没有才回退本地。用于「锁在实例间轮换、新持锁方必须读到上一持锁方
* 写入的基线」这类状态——{@link #get} 的本地优先会让每个实例 forever 读自己的旧值。
*/
public V getFresh(K key) {
if (key == null) {
return null;
}
V remote = readRemote(key);
if (remote != null) {
local.put(key, remote);
return remote;
}
return local.get(key);
}
public boolean containsKey(K key) {
return get(key) != null;
}
@@ -3,7 +3,14 @@ package com.nanri.aiimage.modules.auth.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.auth.model.entity.LoginUserEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@Mapper
public interface LoginUserMapper extends BaseMapper<LoginUserEntity> {
/**
* 会话世代 +1(登录、登出各调用一次)。原子自增而不是「读出来再写回」:
* 双节点同时登录时后者不能把前者的世代覆盖回去。返回 0 行表示用户已不存在。
*/
int incrementSessionEpoch(@Param("userId") Long userId);
}
@@ -43,4 +43,10 @@ public class LoginUserEntity {
*/
@TableField("disabled_at")
private LocalDateTime disabledAt;
/**
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
* 与 {@code AdminUserEntity.sessionEpoch} 同列,改一处必须同改两处。
*/
@TableField("session_epoch")
private Long sessionEpoch;
}
@@ -116,6 +116,7 @@ public class AuthService {
accountExpiryService.assertLoginAllowed(user);
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
advanceSessionEpoch(user);
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
if (authProperties.isSingleDeviceEnabled()) {
DeviceSessionPolicy.logBindOnLogin(user.getMachine(), deviceId, user.getId(), user.getUsername());
@@ -125,7 +126,8 @@ public class AuthService {
} else if (DeviceSessionPolicy.isRevoked(user.getMachine())) {
// 开关关掉后登录不再写 machine,登出留下的吊销标记就没人清:此后每次登录都会先成功、
// 再被 requireUser 的 assertNotRevoked 判成「登录已过期」,账号永久锁死。
// 登录成功即视为新会话,清空标记(空=未绑定,放行语义);仍不写设备号。
// 登录成功即视为新会话(上面已推进 session_epoch,旧令牌另有世代校验兜底),
// 清空标记(空=未绑定,放行语义);仍不写设备号。
log.warn("[auth] 登录清除登出吊销标记 userId={} username={}", user.getId(), user.getUsername());
loginUserMapper.update(null, new LambdaUpdateWrapper<LoginUserEntity>()
.eq(LoginUserEntity::getId, user.getId())
@@ -149,6 +151,9 @@ public class AuthService {
if (userId == null) {
return;
}
// 世代 +1 是真正的吊销;它不依赖 machine 是否等于本次 deviceId,
// 因此 machine 为空(单设备关闭时的常态)也一定留得下吊销效果。
int epochUpdated = loginUserMapper.incrementSessionEpoch(userId);
String marker = DeviceSessionPolicy.REVOKED_PREFIX + userId + "#"
+ java.time.LocalDateTime.now().format(REVOKE_STAMP);
String bound = deviceId == null ? "" : deviceId.trim();
@@ -156,7 +161,8 @@ public class AuthService {
.set(LoginUserEntity::getMachine, marker)
.eq(LoginUserEntity::getId, userId)
.eq(!bound.isEmpty(), LoginUserEntity::getMachine, bound));
log.info("[auth] 登出吊销 userId={} deviceBound={} updated={}", userId, bound, updated);
log.info("[auth] 登出吊销 userId={} deviceBound={} epochUpdated={} markerUpdated={}",
userId, bound, epochUpdated, updated);
}
/** 从令牌里取出身份再吊销;解析不出(已过期/伪造)就什么都不做——cookie 由调用方清。 */
@@ -173,7 +179,20 @@ public class AuthService {
log.warn("[auth] 登出时 subject 非数字,跳过吊销(cookie 仍由调用方清)");
return;
}
logout(Long.parseLong(subject), DeviceSessionPolicy.claimDeviceId(claims));
Long userId = Long.parseLong(subject);
LoginUserEntity current = loginUserMapper.selectById(userId);
if (current == null) {
return;
}
// 旧令牌不能把新会话一起登出:只有当前世代的令牌才有权推进世代。
long claimEpoch = DeviceSessionPolicy.claimSessionEpoch(claims);
long storedEpoch = current.getSessionEpoch() == null ? 0L : current.getSessionEpoch();
if (claimEpoch != storedEpoch) {
log.warn("[auth] 登出令牌的会话世代已过期,跳过吊销 userId={} stored={} claim={}",
userId, storedEpoch, claimEpoch);
return;
}
logout(userId, DeviceSessionPolicy.claimDeviceId(claims));
}
public LoginResultVo checkLogin(String token, String currentDeviceId) {
@@ -196,6 +215,9 @@ public class AuthService {
}
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
String claimDeviceId = DeviceSessionPolicy.claimDeviceId(claims);
// 世代校验先于设备比对:登出/新登录后旧 token 在这里直接 401,不能靠 check_login 续期复活
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
// 单设备登录:不拦截的话,被顶下线的旧 token 会在这里续期「复活」
if (authProperties.isSingleDeviceEnabled()) {
DeviceSessionPolicy.assertSameDevice(user.getMachine(), claimDeviceId,
@@ -238,8 +260,20 @@ public class AuthService {
return authProperties.getCookieName();
}
/** 登录推进世代;返回 0 行说明用户行已消失,不能再签发一个「世代比库还新」的令牌。 */
private void advanceSessionEpoch(LoginUserEntity user) {
int updated = loginUserMapper.incrementSessionEpoch(user.getId());
if (updated != 1) {
log.warn("[auth] 登录推进会话世代失败 userId={} updated={}", user.getId(), updated);
throw new BusinessException(500, "登录状态更新失败,请重试");
}
long current = user.getSessionEpoch() == null ? 0L : user.getSessionEpoch();
user.setSessionEpoch(current + 1);
}
private LoginResultVo buildResult(LoginUserEntity user, String deviceId, boolean isAdmin) {
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId);
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId,
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
LoginResultVo vo = new LoginResultVo();
vo.setUserId(user.getId());
vo.setUsername(user.getUsername());
@@ -57,7 +57,8 @@ public class ClientCredentialController {
vo.setMysqlPassword(blankToEmpty(clientCredentialProperties.getMysqlPassword()));
// 续期与容忍是两件事:容忍让存量客户端不被 7 天边界掐断,这条把新客户端拉回正常轨道
vo.setJwt(jwtService.issue(user.getId(), user.getUsername(),
adminAuthSupport.currentDeviceId(request)));
adminAuthSupport.currentDeviceId(request),
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch()));
// 只记身份不记值:便于事后回答"哪个账号在哪台机器上领过紫鸟口令"
log.info("[client-credentials] 下发凭据 userId={} username={} deviceId={}",
user.getId(), user.getUsername(), adminAuthSupport.currentDeviceId(request));
@@ -22,6 +22,8 @@ public class TaskFileJobEntity {
private String jobType;
private String status;
private Integer retryCount;
/** 卡死扫描的重排次数(V173),与真失败 retry_count 分账;达到上限后扫描侧判死。 */
private Integer stuckRequeueCount;
private String errorMessage;
private String resultFileUrl;
private LocalDateTime createdAt;
@@ -159,6 +159,12 @@ public class TaskDistributedLockService {
return moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
}
/** 同包用例读数:这把锁是否被标成续期丢失(良性停续期必须为 false)。 */
boolean activeRenewalLost(String lockName) {
ActiveTaskLockRegistry.ActiveLock lock = activeLocks.snapshot().get(lockName);
return lock != null && lock.isRenewalLost();
}
/** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
Map<String, Long> activeHeldMillis() {
long now = System.currentTimeMillis();
@@ -327,15 +333,26 @@ public class TaskDistributedLockService {
/**
* 停止看门狗续期,但**不释放**锁:父任务已终态时,本节点不该再给这把锁续命;
* 仍由 {@link #close()} 那次带 token 的 RELEASE 收尾,避免把接管者的锁误删。
*
* <p>默认按「续期丢失」标记,保留给真正的续期失败路径;父任务终态这种良性停续期
* 必须传 {@code markLost=false},否则 {@code _prod_watch} 的 ACTIVE_LOST 会把良性
* 停续期计成异常。
*/
public void stopRenewal(String reason) {
stopRenewal(reason, true);
}
public void stopRenewal(String reason, boolean markLost) {
if (heldLock.closed.compareAndSet(false, true)) {
cancelRenewal(heldLock);
activeLocks.markRenewalLost(heldLock.lockName);
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={}",
if (markLost) {
activeLocks.markRenewalLost(heldLock.lockName);
}
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={} markLost={}",
heldLock.lockName,
System.currentTimeMillis() - heldLock.acquiredAtMillis,
reason);
reason,
markLost);
}
}
@@ -28,6 +28,12 @@ public class TaskFileJobService {
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
public static final int MAX_RETRY_COUNT = 5;
/**
* 卡死重排的独立上限。B28 之后卡死重排不扣 retry_count,若没有上限,一个无异常挂住的
* job 会被无限放回 PENDING,并让 TaskResumeService 永远跳过克隆重试。20 次是
* 「慢执行」与「真卡死」的分界:按 5 分钟 stuck 阈值约 100 分钟,远大于现网合法慢单。
*/
public static final int MAX_STUCK_REQUEUE = 20;
private final TaskFileJobMapper taskFileJobMapper;
private final ApplicationEventPublisher applicationEventPublisher;
@@ -419,16 +425,41 @@ public class TaskFileJobService {
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
// 0 字节收场。卡死次数改用独立列 stuck_requeue_count 记账,达到上限才判死;
// 这样既保住"慢执行不扣预算",又不会让无异常挂住的 job 无限重排。
LocalDateTime now = LocalDateTime.now();
int stuckCount = job.getStuckRequeueCount() == null ? 0 : job.getStuckRequeueCount();
if (stuckCount >= MAX_STUCK_REQUEUE) {
int failed = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.set(TaskFileJobEntity::getStuckRequeueCount, MAX_STUCK_REQUEUE)
.set(TaskFileJobEntity::getErrorMessage,
"文件生成任务连续 " + MAX_STUCK_REQUEUE + " 次卡死重排,已判失败")
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (failed > 0) {
exhaustedJobIds.add(job.getId());
log.warn("[task-file-job][STUCK_REQUEUE_EXHAUSTED] jobId={} taskId={} moduleType={} stuckRequeueCount={}",
job.getId(), job.getTaskId(), job.getModuleType(), stuckCount);
}
continue;
}
int nextStuckCount = stuckCount + 1;
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
.set(TaskFileJobEntity::getStuckRequeueCount, nextStuckCount)
.set(TaskFileJobEntity::getErrorMessage,
"文件生成任务运行超时,已重新排队(第 " + nextStuckCount + " 次)")
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
@@ -24,6 +24,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
@Service
@@ -199,15 +200,26 @@ public class TaskResultFileJobWorker {
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
long interval = Math.max(1000L, heartbeatIntervalMillis);
AtomicBoolean terminalStopped = new AtomicBoolean(false);
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
if (terminalStopped.get()) {
return;
}
try {
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
if (handle != null) {
handle.stopRenewal("parent-task-terminal");
log.info("[task-file-job] stopped lock renewal because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
if (terminalStopped.compareAndSet(false, true)) {
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
if (handle != null) {
// 良性停续期:父任务已终态,不是续期失败,不能打 renewalLost=true 造成假告警
handle.stopRenewal("parent-task-terminal", false);
}
// 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新,
// job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
job.getId(), job.getTaskId(), job.getModuleType());
}
return;
}
taskFileJobService.touchRunning(job.getId());
} catch (Exception ex) {
@@ -15,6 +15,11 @@ public class CostGuardStatusVo {
private LocalDateTime pausedAt;
private TodayUsage today;
private WindowUsage window;
/**
* window 是本实例最近一轮扫描的读数,不是集群全局值。前端据此标注「本节点读数」,
* 避免把单个实例的窗口值误读成全局速率。
*/
private boolean windowLocalOnly;
private Thresholds thresholds;
private List<SharedSecretWarningVo> sharedWarnings;
@@ -0,0 +1,66 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.NodeSharedStore;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
/**
* 损耗扫描基线(用户 → 当日累计三元组 + 采样时刻),跨实例共享。
*
* <p>此前基线是单实例内存,而扫描锁是集群级:持锁方一轮换,新持锁方拿到的
* {@code before} 可能是几分钟前的,增量会被当成 60 秒窗口,导致误暂停;反过来新实例
* 首轮只建基线,又会漏掉一次窗口。基线落 Redis 后,任何实例接手都按真实 elapsed 归一化。
*
* <p>用 {@link NodeSharedStore#getFresh} 而不是 {@code get}:本地缓存优先会让每个实例
* 永远读自己的旧基线,正好是这次要修的问题。
*/
@Service
public class CostGuardBaselineStore {
private static final Duration TTL = Duration.ofHours(26);
private final NodeSharedStore<Long, Baseline> store;
public CostGuardBaselineStore(StringRedisTemplate redis, ObjectMapper objectMapper) {
this.store = new NodeSharedStore<>("cost-guard:baseline", TTL, Baseline.class,
redis, objectMapper, 0L);
}
/** 测试用:只传一个共享 store,两个调度器实例即可模拟跨实例读数。 */
CostGuardBaselineStore(NodeSharedStore<Long, Baseline> store) {
this.store = store;
}
public Baseline get(Long userId) {
return store.getFresh(userId);
}
public void put(Long userId, Baseline baseline) {
store.putNow(userId, baseline);
}
public void remove(Long userId) {
store.remove(userId);
}
/** Jackson 需要无参构造;字段公开以保持序列化形状稳定。 */
public static class Baseline {
public long proxy;
public long llmCalls;
public long llmTokens;
public long capturedAtMillis;
public Baseline() {
}
public Baseline(long proxy, long llmCalls, long llmTokens, long capturedAtMillis) {
this.proxy = proxy;
this.llmCalls = llmCalls;
this.llmTokens = llmTokens;
this.capturedAtMillis = capturedAtMillis;
}
}
}
@@ -28,7 +28,10 @@ import java.util.concurrent.ConcurrentHashMap;
* 每分钟全量 GROUP BY 的成本可以忽略。
*
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
* 基线对齐到当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
*
* <p>基线落在 {@link CostGuardBaselineStore}(Redis),不是单实例内存:扫描锁是集群级的,
* 持锁方轮换后新实例必须按真实 elapsed 归一化增量,否则会把多分钟增量当成 60 秒窗口误暂停。
*/
@Service
@RequiredArgsConstructor
@@ -43,10 +46,11 @@ public class CostGuardScheduler {
private final SharedSecretMonitorService sharedSecretMonitorService;
private final NotificationDispatchService notificationDispatchService;
private final DistributedJobLockService distributedJobLockService;
private final CostGuardBaselineStore baselineStore;
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
private volatile long lastSharedRefreshAt = 0L;
private volatile long lastSharedNotifyAt = 0L;
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
@@ -56,6 +60,9 @@ public class CostGuardScheduler {
}
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
if (lock == null) {
// 非持锁方也要刷新共享密钥快照:否则本实例 status() 永远读到空 warningsByUser,
// 用户看到的「没有共用」与真实情况相反。通知只由持锁方发(见 runOnce)。
maybeRefreshShared(false);
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
return;
}
@@ -68,53 +75,70 @@ public class CostGuardScheduler {
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
void runOnce() {
long nowMillis = System.currentTimeMillis();
int windowSeconds = Math.max(60, properties.getWindowSeconds());
Map<Long, UserTotals> current = loadTodayTotals();
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
int paused = 0;
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
Long userId = entry.getKey();
UserTotals now = entry.getValue();
UserTotals before = previousTotals.get(userId);
CostGuardBaselineStore.Baseline before = baselineStore.get(userId);
boolean advanceBaseline = true;
if (before == null) {
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
// 首轮:窗口增量不判(启动前的大批次不是分钟增量),但日累计已经越线必须暂停,
// 否则重启/首次部署后当天永远不会被拦。
deltas.put(userId, WindowDelta.ZERO);
continue;
if (!guardService.isPaused(userId)) {
String reason = dailyBreachReason(now);
if (reason != null && !pauseAndNotify(userId, reason)) {
// 写失败不推进基线:下一轮仍按「已越线」重试,否则 crossed(before, now)
// 会因 before 被抬到阈值之上而永远不再触发。
advanceBaseline = false;
} else if (reason != null) {
paused++;
}
}
} else {
WindowDelta raw = new WindowDelta(
Math.max(0, now.proxy - before.proxy),
Math.max(0, now.llmCalls - before.llmCalls),
Math.max(0, now.llmTokens - before.llmTokens));
long elapsedSeconds = Math.max(1L, (nowMillis - before.capturedAtMillis) / 1000L);
WindowDelta windowDelta = normalize(raw, elapsedSeconds, windowSeconds);
deltas.put(userId, windowDelta);
if (!guardService.isPaused(userId)) {
String reason = breachReason(now, before, windowDelta, windowSeconds);
if (reason != null && !pauseAndNotify(userId, reason)) {
advanceBaseline = false;
} else if (reason != null) {
paused++;
}
}
}
WindowDelta delta = new WindowDelta(
Math.max(0, now.proxy - before.proxy),
Math.max(0, now.llmCalls - before.llmCalls),
Math.max(0, now.llmTokens - before.llmTokens));
deltas.put(userId, delta);
if (guardService.isPaused(userId)) {
continue;
if (advanceBaseline) {
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
now.proxy, now.llmCalls, now.llmTokens, nowMillis));
}
String reason = breachReason(now, before, delta);
if (reason == null) {
continue;
}
guardService.pause(userId, reason);
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
null);
paused++;
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
}
previousTotals.clear();
previousTotals.putAll(current);
lastDeltas.clear();
lastDeltas.putAll(deltas);
maybeRefreshShared();
maybeRefreshShared(true);
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
}
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
/**
* 用户点击继续时调用:把基线对齐到当前值(而不是删掉基线)。删掉会让下一轮走「首轮」
* 分支;首轮现在会按日累计判暂停,用户刚点继续就会被立刻再暂停。
*/
public void resetBaseline(Long userId) {
if (userId == null) {
return;
}
previousTotals.remove(userId);
UserTotals current = todayTotals(userId);
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
current.proxy, current.llmCalls, current.llmTokens, System.currentTimeMillis()));
lastDeltas.remove(userId);
}
@@ -158,21 +182,35 @@ public class CostGuardScheduler {
return totals;
}
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
int seconds = Math.max(60, properties.getWindowSeconds());
/** 暂停写成功才发通知、才计数:写失败时谎报「已暂停」比不暂停更糟。 */
private boolean pauseAndNotify(Long userId, String reason) {
if (!guardService.pause(userId, reason)) {
log.warn("[cost-guard] 暂停写入失败,本轮不通知 userId={} reason={}", userId, reason);
return false;
}
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
null);
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
return true;
}
private String breachReason(UserTotals now, CostGuardBaselineStore.Baseline before,
WindowDelta delta, int windowSeconds) {
if (properties.getProxyWindowThreshold() > 0
&& delta.proxy >= properties.getProxyWindowThreshold()) {
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
&& delta.proxy() >= properties.getProxyWindowThreshold()) {
return "最近约 " + windowSeconds + " 秒代理提取 " + delta.proxy() + " 次,超过阈值 "
+ properties.getProxyWindowThreshold() + " 次";
}
if (properties.getLlmWindowCallsThreshold() > 0
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
&& delta.llmCalls() >= properties.getLlmWindowCallsThreshold()) {
return "最近约 " + windowSeconds + " 秒 LLM 调用 " + delta.llmCalls() + " 次,超过阈值 "
+ properties.getLlmWindowCallsThreshold() + " 次";
}
if (properties.getLlmWindowTokensThreshold() > 0
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
&& delta.llmTokens() >= properties.getLlmWindowTokensThreshold()) {
return "最近约 " + windowSeconds + " 秒 LLM 消耗 " + delta.llmTokens() + " tokens,超过阈值 "
+ properties.getLlmWindowTokensThreshold();
}
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
@@ -190,18 +228,73 @@ public class CostGuardScheduler {
return null;
}
/** 首轮专用:没有上一轮基线可比,只按「已经越线」判日累计,不判窗口增量。 */
private String dailyBreachReason(UserTotals now) {
if (properties.getDailyProxyThreshold() > 0
&& now.proxy >= properties.getDailyProxyThreshold()) {
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
+ properties.getDailyProxyThreshold() + " 次";
}
if (properties.getDailyLlmCallsThreshold() > 0
&& now.llmCalls >= properties.getDailyLlmCallsThreshold()) {
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
+ properties.getDailyLlmCallsThreshold() + " 次";
}
if (properties.getDailyLlmTokensThreshold() > 0
&& now.llmTokens >= properties.getDailyLlmTokensThreshold()) {
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
+ properties.getDailyLlmTokensThreshold();
}
return null;
}
private boolean crossed(long before, long now, long threshold) {
return threshold > 0 && before < threshold && now >= threshold;
}
private void maybeRefreshShared() {
/**
* 把「elapsed 秒的真实增量」换算成 60 秒窗口。持锁方轮换、扫描被拖长、进程重启后
* 首轮接手都会让 elapsed > windowSeconds;不归一化就是把多分钟增量当一分钟判。
*
* <p>只向下缩放(elapsed 大于窗口时),elapsed 小于窗口时直接采用原始增量:
* 短间隔的原始增量是保守下界,放大它只会在调度抖动时制造误暂停。
*/
private WindowDelta normalize(WindowDelta raw, long elapsedSeconds, int windowSeconds) {
if (elapsedSeconds <= windowSeconds) {
return raw;
}
return new WindowDelta(
scaleToWindow(raw.proxy(), elapsedSeconds, windowSeconds),
scaleToWindow(raw.llmCalls(), elapsedSeconds, windowSeconds),
scaleToWindow(raw.llmTokens(), elapsedSeconds, windowSeconds));
}
private long scaleToWindow(long value, long elapsedSeconds, int windowSeconds) {
if (value <= 0) {
return 0;
}
// 向上取整:宁可早一秒判红,也不把刚好越线的批次放过
return Math.max(1L, (value * windowSeconds + elapsedSeconds - 1) / elapsedSeconds);
}
/**
* 共享密钥快照按 shared-scan-interval-ms 刷新。通知只由持锁方发,避免三台实例各推一遍;
* 但非持锁方也要刷新快照,否则它的 status() 永远看不到共享提示。
*/
private void maybeRefreshShared(boolean notify) {
long now = System.currentTimeMillis();
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
long interval = Math.max(60_000L, properties.getSharedScanIntervalMs());
boolean refreshDue = now - lastSharedRefreshAt >= interval;
boolean notifyDue = notify && now - lastSharedNotifyAt >= interval;
if (!refreshDue && !notifyDue) {
return;
}
lastSharedRefreshAt = now;
// 不自己吞:runOnce 里这是最后一步,抛出由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
sharedSecretMonitorService.refresh();
if (notifyDue) {
lastSharedNotifyAt = now;
}
// 不自己吞:refresh 失败由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
sharedSecretMonitorService.refresh(notifyDue);
}
public static final class UserTotals {
@@ -30,6 +30,7 @@ public class CostGuardService {
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
vo.setWindowLocalOnly(true);
vo.setThresholds(new CostGuardStatusVo.Thresholds(
properties.getProxyWindowThreshold(),
properties.getLlmWindowCallsThreshold(),
@@ -49,6 +49,14 @@ public class SharedSecretMonitorService {
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
public synchronized void refresh() {
refresh(true);
}
/**
* @param notify 是否推当日提示。多实例下只有持锁方传 true,其余实例只刷新快照,
* 否则每台实例各推一遍(虽然通知有去重键,但仍是三倍写入与扇出)。
*/
public synchronized void refresh(boolean notify) {
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
warningsByUser = Map.of();
snapshotAtMillis = System.currentTimeMillis();
@@ -89,7 +97,9 @@ public class SharedSecretMonitorService {
type, group.module.label(), accounts, threshold, message);
for (Long userId : group.userIds) {
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
pushDailyWarning(userId, group.module, entry.getKey(), warning);
if (notify) {
pushDailyWarning(userId, group.module, entry.getKey(), warning);
}
}
}
warningsByUser = next;
@@ -55,19 +55,25 @@ public class UserTaskGuardService {
return state;
}
public void pause(Long userId, String reason) {
/**
* @return 写入是否成功。调用方只有拿到 true 才能发「已暂停」通知;
* 此前该方法吞掉异常,扫描器无论成败都通知,数据库抖动时用户被告知已暂停、任务却仍在派发。
*/
public boolean pause(Long userId, String reason) {
if (userId == null || userId <= 0) {
return;
return false;
}
String safeReason = truncate(reason);
try {
guardMapper.pause(userId, safeReason);
cache.remove(userId);
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
return true;
} catch (Exception ex) {
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
userId, safeReason, ex.getMessage(), ex);
return false;
}
}
@@ -18,6 +18,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Pattern;
/**
@@ -117,13 +118,16 @@ public class WorkbenchClientPackageService {
}
/**
* 签发直传预签名:对象 key 由「毫秒时间戳 + 安全版本号」组成,每次上传互不覆盖。
* 签发直传预签名:对象 key 由「毫秒时间戳 + 随机 UUID + 安全版本号」组成。
* UUID 是 CR-08 的修复点:只靠毫秒时间戳时,多实例时钟相同或同毫秒两名管理员
* 上传同一版本会生成同一个 key,后一次 PUT 覆盖前一次对象、先确认的 DB 行摘要失效。
*/
public Map<String, Object> presignPackage(String fileName, String version) {
String normalizedName = requireZipFileName(fileName);
String normalizedVersion = requireVersion(version);
String objectKey = OBJECT_KEY_PREFIX
+ OBJECT_NAME_FORMATTER.format(LocalDateTime.now())
+ "-" + UUID.randomUUID().toString().replace("-", "")
+ "-" + safeVersionKey(normalizedVersion) + ".zip";
log.info("[workbench-client] 签发客户端包直传预签名 version={} fileName={} objectKey={} expirySeconds={}",
normalizedVersion, normalizedName, objectKey, PRESIGN_EXPIRY_SECONDS);
@@ -154,27 +158,44 @@ public class WorkbenchClientPackageService {
normalizedVersion, objectKey);
throw new BusinessException("版本号与上传对象不匹配,请重新上传");
}
// 幂等:重复 confirm 同一对象返回已登记行,不再报错(CR-08)
WorkbenchClientPackageEntity existing = packageMapper.selectOne(
new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey)
.last("LIMIT 1"));
if (existing != null) {
log.info("[workbench-client] 直传确认幂等命中 id={} objectKey={}", existing.getId(), objectKey);
return toItemMap(existing);
}
long size = ossStorageService.workbenchClientObjectSize(objectKey);
if (size < 0) {
log.warn("[workbench-client] 直传确认失败:对象不存在 objectKey={}", objectKey);
throw new BusinessException("对象存储中未找到上传的安装包,请重新上传");
}
if (size <= 0) {
// 0 字节对象(空文件/中断上传)会被客户端拿到后解压失败,直接拒绝并清残
log.warn("[workbench-client] 直传对象为空,清理残留 objectKey={}", objectKey);
ossStorageService.removeWorkbenchClientObject(objectKey);
throw new BusinessException("上传的安装包为空文件,请重新上传");
}
if (size > MAX_UPLOAD_BYTES) {
log.warn("[workbench-client] 直传对象超过大小上限,清理残留 objectKey={} bytes={}", objectKey, size);
ossStorageService.removeWorkbenchClientObject(objectKey);
throw new BusinessException("文件超过允许的大小限制");
}
Long sameVersion = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.eq(WorkbenchClientPackageEntity::getVersion, normalizedVersion));
if (sameVersion != null && sameVersion > 0) {
// 同版本不同字节会让已升级的装机按版本号比较后不升级,客户端还会拿到错误摘要
log.warn("[workbench-client] 直传确认失败:版本已存在 version={} objectKey={}",
normalizedVersion, objectKey);
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
}
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
if (sha256 == null || sha256.length() != 64) {
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
throw new BusinessException("安装包摘要计算失败,请重新上传");
}
Long duplicated = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey));
if (duplicated != null && duplicated > 0) {
log.warn("[workbench-client] 直传确认失败:对象已被记录,重复提交 objectKey={}", objectKey);
throw new BusinessException("该安装包已登记,请勿重复提交");
}
WorkbenchClientPackageEntity entity = new WorkbenchClientPackageEntity();
entity.setVersion(normalizedVersion);
@@ -0,0 +1,23 @@
-- =============================================================
-- 迁移 V172__user_session_epoch
-- 立账:ops 仓 docs/audit/CODE_REVIEW_REPORT.md CR-01(2026-10-25)
--
-- 1. 变更目的:给 users 增加会话世代。旧 JWT 没有服务端黑名单,登出只靠 users.machine
-- 上的 revoked@ 标记失效;单设备开关关闭时登录会把这个标记清空,旧令牌重新可用。
-- 世代随登录、登出各 +1,令牌签发时写入 epoch claim,鉴权时与库中值比对,
-- 旧令牌在登出后、以及在任何一次新登录后都立即失效。
--
-- 2. 口径:
-- - 列 NOT NULL DEFAULT 0;存量用户为 0,升级前签发的旧令牌没有 epoch claim 时
-- 按 0 读取,因此不会被本次上线统一踢下线;用户下一次登录/登出推进后旧令牌失效。
-- - machine 的互踢逻辑保持不变;单设备开关关闭时不再依赖清空 revoked@ 标记表达新会话。
--
-- 3. 影响表与数据量:users 加一列,存量行默认 0;MySQL 8.0 起 ADD COLUMN 走 INSTANT,
-- 不重建表。本机无生产库连接,未做上线前 dry-run;上线后由 Flyway 自动应用。
--
-- 4. 回滚(如需):ALTER TABLE `users` DROP COLUMN `session_epoch`;
-- 回滚前需同步回滚应用代码,否则新代码读不到列会启动失败(Expand-and-Contract 的反向)。
-- =============================================================
ALTER TABLE `users`
ADD COLUMN `session_epoch` BIGINT NOT NULL DEFAULT 0 COMMENT '会话世代:登录/登出各 +1' AFTER `machine`;
@@ -0,0 +1,24 @@
-- =============================================================
-- 迁移 V173__task_file_job_stuck_requeue_count
-- 立账:ops 仓 docs/audit/CODE_REVIEW_REPORT.md CR-03(2026-10-25)
--
-- 1. 变更目的:给卡死重排单独记账。B28 拍板「RUNNING 卡死重排不消耗 retry_count」
-- 之后,一个无异常挂住的 job 会被无限放回 PENDING,并且 TaskResumeService 会因此
-- 永远跳过克隆重试。本列只统计 stuck 扫描的重排次数,与真失败重试预算分账;
-- 达到上限(20)时扫描侧直接判 FAILED,保留「慢执行不扣预算」的口径。
--
-- 2. 口径:
-- - NOT NULL DEFAULT 0,存量行从 0 起算;
-- - 只由 TaskFileJobService.resetStuckRunningJobsDetailed 递增;
-- - 开机回收(reclaimOwnedRunningJobs)不递增:发版不是作业的错。
--
-- 3. 影响表与数据量:biz_task_file_job 加一列,存量行默认 0;MySQL 8.0 起 ADD COLUMN
-- 走 INSTANT,不重建表。未连生产库 dry-run,随下一次发版由 Flyway 自动应用。
--
-- 4. 回滚(如需):ALTER TABLE `biz_task_file_job` DROP COLUMN `stuck_requeue_count`;
-- 回滚前需同步回滚应用代码。
-- =============================================================
ALTER TABLE `biz_task_file_job`
ADD COLUMN `stuck_requeue_count` INT NOT NULL DEFAULT 0 COMMENT '卡死重排次数(独立于 retry_count)'
AFTER `retry_count`;
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.nanri.aiimage.modules.auth.mapper.LoginUserMapper">
<update id="incrementSessionEpoch">
UPDATE users
SET session_epoch = session_epoch + 1
WHERE id = #{userId}
</update>
</mapper>
@@ -101,7 +101,9 @@ class MigratedSqlEquivalenceTest {
"UserSecretUsageMapper#sumByUserAndModule",
"UserSecretUsageMapper#sumForUser",
// B28(2026-10-25)新写的 1 条主键点查:父任务是否已终态。
"TaskFileJobMapper#selectFileTaskStatus");
"TaskFileJobMapper#selectFileTaskStatus",
// CR-01(2026-10-25)新写的 1 条写侧语句:会话世代原子自增。
"LoginUserMapper#incrementSessionEpoch");
/** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */
private static final int LEDGER_COLUMNS = 5;
@@ -80,7 +80,9 @@ class SqlLayeringConventionTest {
// 126 → 128:同轮 {@code UserTaskGuardMapper} 的 pause/resume 两条写侧语句从注解迁进 XML。
// 128 → 129:B28 新增 {@code TaskFileJobMapper#selectFileTaskStatus}(按主键读父任务状态),
// 新写语句按约定直接进 XML,同轮登记进 MigratedSqlEquivalenceTest.NOT_MIGRATED(88 → 89)。
private static final int XML_STATEMENT_TOTAL = 129;
// 129 → 130:CR-01 新增 {@code LoginUserMapper#incrementSessionEpoch}(会话世代原子自增),
// 新写语句直接进 mapper/auth/LoginUserMapper.xml,同轮登记进 NOT_MIGRATED。
private static final int XML_STATEMENT_TOTAL = 130;
/**
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
@@ -126,6 +126,70 @@ class ExcelStreamReaderParseFailureTest {
assertEquals("Nike", seen.toString());
}
@Test
void ooxmlCallsSheetTotalAfterHeader() throws Exception {
Path good = writeWorkbook();
java.util.List<String> events = new java.util.ArrayList<>();
ExcelStreamReader.readFirstSheet(good.toFile(), new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
events.add("header:" + headerMap.get(0));
}
@Override
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
events.add("total:" + approximateTotalRows);
}
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
}
});
assertEquals("header:品牌", events.get(0));
assertTrue(events.size() >= 2 && events.get(1).startsWith("total:"),
"onSheetTotal 必须在 onHeader 之后回调,实际=" + events);
}
@Test
void hssfCallsSheetTotalAfterHeader() throws Exception {
Path file = tempDir.resolve("order.xls");
try (HSSFWorkbook workbook = new HSSFWorkbook()) {
Sheet sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("品牌");
Row data = sheet.createRow(1);
data.createCell(0).setCellValue("Nike");
try (OutputStream out = Files.newOutputStream(file)) {
workbook.write(out);
}
}
java.util.List<String> events = new java.util.ArrayList<>();
ExcelStreamReader.readFirstSheet(file.toFile(), new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
events.add("header:" + headerMap.get(0));
}
@Override
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
events.add("total:" + approximateTotalRows);
}
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
}
});
assertEquals("header:品牌", events.get(0));
assertTrue(events.size() >= 2 && events.get(1).startsWith("total:"),
"xls 路径同样必须先表头、后总行数,实际=" + events);
}
private Path writeWorkbook() throws IOException {
Path file = tempDir.resolve("ok.xlsx");
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
@@ -96,6 +96,20 @@ class AdminAuthSupportTest {
assertThatCode(() -> support.requireUser(requestWithBearer("t"))).doesNotThrowAnyException();
}
@Test
void requireUserRejectsTokenFromBeforeLogoutOrRelogin() {
// 库中世代已是 2,token 还签着 1:登出/新登录之后旧令牌必须 401,与单设备开关无关
AdminUserEntity user = user(7L, "normal", 0, null, null);
user.setSessionEpoch(2L);
AdminAuthSupport support = support(user, "devA", false, 1L);
BusinessException ex = catchThrowableOfType(
() -> support.requireUser(requestWithBearer("t")), BusinessException.class);
assertThat(ex.getCode()).isEqualTo(401);
assertThat(ex.getMessage()).contains("登录已过期");
}
// ---------- 内部令牌:轮换过渡期新旧两把同时接受 ----------
@Test
@@ -199,6 +213,12 @@ class AdminAuthSupportTest {
}
private AdminAuthSupport support(AdminUserEntity user, String claimDeviceId, boolean singleDeviceEnabled) {
return support(user, claimDeviceId, singleDeviceEnabled,
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
}
private AdminAuthSupport support(AdminUserEntity user, String claimDeviceId, boolean singleDeviceEnabled,
long claimSessionEpoch) {
JwtService jwtService = mock(JwtService.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
AuthProperties props = mock(AuthProperties.class);
@@ -206,6 +226,7 @@ class AdminAuthSupportTest {
Claims claims = mock(Claims.class);
when(claims.getSubject()).thenReturn(String.valueOf(user.getId()));
when(claims.get("deviceId")).thenReturn(claimDeviceId);
when(claims.get("sessionEpoch")).thenReturn(claimSessionEpoch);
when(jwtService.parse("t")).thenReturn(claims);
when(jwtService.parseWithVerdict("t")).thenReturn(
new JwtService.ParseResult(JwtService.Verdict.VALID, claims));
@@ -37,7 +37,7 @@ class JwtVerdictTest {
@Test
void freshTokenIsValidAndParseStillReturnsClaims() {
JwtService s = service(SECRET);
String token = s.issue(7L, "小魏16", "dev-1");
String token = s.issue(7L, "小魏16", "dev-1", 3L);
JwtService.ParseResult result = s.parseWithVerdict(token);
assertThat(result.verdict()).isEqualTo(JwtService.Verdict.VALID);
assertThat(result.isValid()).isTrue();
@@ -59,7 +59,7 @@ class JwtVerdictTest {
@Test
void wrongKeyIsInvalidNotExpired() {
String forged = service("9999999999012345678901234567890123456789")
.issue(7L, "小魏16", "dev-1");
.issue(7L, "小魏16", "dev-1", 3L);
JwtService.ParseResult result = service(SECRET).parseWithVerdict(forged);
assertThat(result.verdict()).isEqualTo(JwtService.Verdict.INVALID);
assertThat(result.claims()).isNull();
@@ -11,6 +11,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -91,6 +92,14 @@ class MigrationInventoryTest {
*/
private static final Set<Integer> RETIRED_HOLES = Set.of(132, 133, 148, 168, 169);
/**
* 文件名版本与头注释版本不一致的历史件。V170 的头注释误写成 V168;它已应用到生产,
* 按 AGENTS.md「已发布迁移一个字节都不许改」不能回改(CR-13)。文件名才是 Flyway
* 认的号,这里把它显式登记成唯一豁免;新增任何不一致都会判红。
*/
private static final Map<String, String> KNOWN_HEADER_VERSION_MISMATCH =
Map.of("V170__user_task_guard.sql", "V168");
@Test
void integerVersionsSequentialFromOne() throws IOException {
List<Migration> all = parseAll(versionFiles());
@@ -144,6 +153,24 @@ class MigrationInventoryTest {
@Test
void headerVersionMatchesFileNameExceptKnownImmutableMismatch() throws IOException {
Map<String, String> mismatches = new LinkedHashMap<>();
Pattern fileNameVersion = Pattern.compile("^V(\\d+)");
Pattern headerVersion = Pattern.compile("(?m)^--\\s*迁移\\s+V(\\d+)");
for (Path file : versionFiles()) {
String name = file.getFileName().toString();
Matcher fileMatcher = fileNameVersion.matcher(name);
Matcher headerMatcher = headerVersion.matcher(Files.readString(file));
if (fileMatcher.find() && headerMatcher.find()
&& !fileMatcher.group(1).equals(headerMatcher.group(1))) {
mismatches.put(name, "V" + headerMatcher.group(1));
}
}
assertEquals(KNOWN_HEADER_VERSION_MISMATCH, mismatches,
"迁移头注释的版本号必须与文件名一致;V170 是已应用件,只能登记为豁免,不能回改");
}
@Test
void checksumStableAcrossReads() throws IOException {
Map<String, String> first = sha256All(versionFiles());
@@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
@@ -57,6 +58,7 @@ class AuthServiceCompanyTest {
// 本组用例只关心公司身份,单设备互踢那条路径留给 AuthServiceTest
when(authProperties.isSingleDeviceEnabled()).thenReturn(false);
when(jwtService.ttlSeconds()).thenReturn(604800L);
when(loginUserMapper.incrementSessionEpoch(7L)).thenReturn(1);
service = new AuthService(loginUserMapper, passwordEncoder, jwtService,
mock(PermissionMenuService.class), authProperties, companyService,
mock(AccountExpiryService.class));
@@ -85,7 +87,7 @@ class AuthServiceCompanyTest {
private void stubSuccessfulLogin(LoginUserEntity user) {
when(loginUserMapper.selectOne(any())).thenReturn(user);
when(passwordEncoder.matches("pwd", "hash")).thenReturn(true);
when(jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
}
@Test
@@ -137,7 +139,7 @@ class AuthServiceCompanyTest {
when(claims.get("deviceId")).thenReturn("devA");
when(jwtService.parse("t")).thenReturn(claims);
when(loginUserMapper.selectById(7L)).thenReturn(user);
when(jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
when(companyService.nameOrEmpty(5L)).thenReturn("厦门米椿花");
LoginResultVo vo = service.checkLogin("t", null);
@@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.catchThrowableOfType;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
@@ -48,7 +49,7 @@ class AuthServiceTest {
Fixture f = fixture(true);
when(f.loginUserMapper.selectOne(any())).thenReturn(f.user("devA"));
when(f.passwordEncoder.matches("pwd", "hash")).thenReturn(true);
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
LoginRequest request = new LoginRequest();
request.setUsername("u7");
@@ -59,7 +60,9 @@ class AuthServiceTest {
// 绑定写入当前设备(last-login-wins),签发的 token 也用当前设备
verify(f.loginUserMapper, times(1)).update(any(), any());
verify(f.jwtService).issue(7L, "u7", "devB");
verify(f.loginUserMapper).incrementSessionEpoch(7L);
// 登录推进世代(0 → 1),新令牌必须签上推进后的值
verify(f.jwtService).issue(7L, "u7", "devB", 1L);
assertThat(vo.getDeviceId()).isEqualTo("devB");
}
@@ -68,7 +71,7 @@ class AuthServiceTest {
Fixture f = fixture(false);
when(f.loginUserMapper.selectOne(any())).thenReturn(f.user("devA"));
when(f.passwordEncoder.matches("pwd", "hash")).thenReturn(true);
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
LoginRequest request = new LoginRequest();
request.setUsername("u7");
@@ -87,7 +90,7 @@ class AuthServiceTest {
Fixture f = fixture(false);
when(f.loginUserMapper.selectOne(any())).thenReturn(f.user("revoked@7#20261024185618"));
when(f.passwordEncoder.matches("pwd", "hash")).thenReturn(true);
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
LoginRequest request = new LoginRequest();
request.setUsername("u7");
@@ -115,7 +118,7 @@ class AuthServiceTest {
assertThat(ex.getCode()).isEqualTo(4011);
assertThat(ex.getMessage()).contains("已在其他设备登录");
// 关键:被踢的旧 token 不能在这里换到新 token「复活」
verify(f.jwtService, never()).issue(any(), anyString(), anyString());
verify(f.jwtService, never()).issue(any(), anyString(), anyString(), anyLong());
}
@Test
@@ -124,11 +127,11 @@ class AuthServiceTest {
Claims claims = f.claims("devA");
when(f.jwtService.parse("t")).thenReturn(claims);
when(f.loginUserMapper.selectById(7L)).thenReturn(f.user("devA"));
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
LoginResultVo vo = f.service.checkLogin("t", null);
verify(f.jwtService).issue(7L, "u7", "devA");
verify(f.jwtService).issue(7L, "u7", "devA", 0L);
assertThat(vo.getDeviceId()).isEqualTo("devA");
}
@@ -155,7 +158,7 @@ class AuthServiceTest {
root.setRole("super_admin");
root.setIsAdmin(1);
when(f.loginUserMapper.selectById(7L)).thenReturn(root);
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
assertThatCode(() -> f.service.checkLogin("t", null)).doesNotThrowAnyException();
}
@@ -166,11 +169,59 @@ class AuthServiceTest {
Claims claims = f.claims("devB");
when(f.jwtService.parse("t")).thenReturn(claims);
when(f.loginUserMapper.selectById(7L)).thenReturn(f.user("devA"));
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
assertThatCode(() -> f.service.checkLogin("t", null)).doesNotThrowAnyException();
}
// ---------- 会话世代:登出/新登录必须让旧 token 失效 ----------
@Test
void checkLoginRejectsTokenFromBeforeLogoutOrRelogin() {
Fixture f = fixture(false);
LoginUserEntity user = f.user("");
user.setSessionEpoch(2L);
Claims claims = f.claims("devA", 1L);
when(f.jwtService.parse("old-token")).thenReturn(claims);
when(f.loginUserMapper.selectById(7L)).thenReturn(user);
BusinessException ex = catchThrowableOfType(
() -> f.service.checkLogin("old-token", null), BusinessException.class);
assertThat(ex.getCode()).isEqualTo(401);
assertThat(ex.getMessage()).contains("登录已过期");
verify(f.jwtService, never()).issue(any(), anyString(), anyString(), anyLong());
}
@Test
void logoutAdvancesSessionEpochEvenWhenMachineIsEmpty() {
// 单设备关闭时 machine 往往为空:旧实现按 machine 做 CAS,命中 0 行 = 登出没留下吊销。
Fixture f = fixture(false);
Claims claims = f.claims("devA", 0L);
when(f.jwtService.parse("t")).thenReturn(claims);
when(f.loginUserMapper.selectById(7L)).thenReturn(f.user(""));
f.service.logoutByToken("t");
verify(f.loginUserMapper).incrementSessionEpoch(7L);
}
@Test
void staleTokenCannotLogOutANewerSession() {
// 旧令牌(世代 1)不能把已经登录到世代 2 的新会话一起登出
Fixture f = fixture(false);
LoginUserEntity user = f.user("");
user.setSessionEpoch(2L);
Claims claims = f.claims("devA", 1L);
when(f.jwtService.parse("t")).thenReturn(claims);
when(f.loginUserMapper.selectById(7L)).thenReturn(user);
f.service.logoutByToken("t");
verify(f.loginUserMapper, never()).incrementSessionEpoch(anyLong());
verify(f.loginUserMapper, never()).update(any(), any());
}
// ---------- 夹具 ----------
private Fixture fixture(boolean singleDeviceEnabled) {
@@ -181,6 +232,7 @@ class AuthServiceTest {
AuthProperties authProperties = mock(AuthProperties.class);
when(authProperties.isSingleDeviceEnabled()).thenReturn(singleDeviceEnabled);
when(jwtService.ttlSeconds()).thenReturn(604800L);
when(loginUserMapper.incrementSessionEpoch(7L)).thenReturn(1);
AuthService service = new AuthService(loginUserMapper, passwordEncoder, jwtService,
permissionMenuService, authProperties, mock(CompanyService.class),
mock(AccountExpiryService.class));
@@ -202,9 +254,14 @@ class AuthServiceTest {
}
Claims claims(String deviceId) {
return claims(deviceId, 0L);
}
Claims claims(String deviceId, long sessionEpoch) {
Claims claims = mock(Claims.class);
when(claims.getSubject()).thenReturn("7");
when(claims.get("deviceId")).thenReturn(deviceId);
when(claims.get("sessionEpoch")).thenReturn(sessionEpoch);
return claims;
}
}
@@ -49,7 +49,7 @@ class ClientCredentialRenewalTest {
when(adminAuthSupport.requireUser(any())).thenReturn(user);
when(adminAuthSupport.currentInternalToken()).thenReturn("token-x");
when(adminAuthSupport.currentDeviceId(any())).thenReturn("dev-1");
when(jwtService.issue(eq(7L), eq("小魏16"), eq("dev-1"))).thenReturn("jwt-fresh");
when(jwtService.issue(eq(7L), eq("小魏16"), eq("dev-1"), eq(0L))).thenReturn("jwt-fresh");
configureAll();
ApiResponse<ClientCredentialVo> response = newController().credentials(request);
assertThat(response.isSuccess()).isTrue();
@@ -19,6 +19,25 @@ import static org.mockito.Mockito.when;
*/
class TaskDistributedLockServiceStopRenewalTest {
@Test
void benignTerminalStopRenewalDoesNotMarkRenewalLost() throws Exception {
// 父任务终态是良性停续期;标成 renewalLost=true 会让 _prod_watch 的 ACTIVE_LOST 误报
DistributedJobLockService delegate = mock(DistributedJobLockService.class);
DistributedJobLockService.LockHandle inner = mock(DistributedJobLockService.LockHandle.class);
when(inner.renew(any(Duration.class))).thenReturn(true);
when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(inner);
TaskDistributedLockService service = new TaskDistributedLockService(delegate);
TaskDistributedLockService.LockHandle handle =
service.acquire("appearance-patent", 32220L, Duration.ofSeconds(1), 0);
handle.stopRenewal("parent-task-terminal", false);
assertEquals(false, service.activeRenewalLost("task:appearance-patent:32220"),
"终态停续期不是续期失败");
handle.close();
}
@Test
void stopRenewalStopsTheWatchdogWithoutReleasingTheLock() throws Exception {
AtomicInteger renewCalls = new AtomicInteger();
@@ -83,6 +83,51 @@ class TaskFileJobServiceTest {
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
}
@Test
void stuckRequeueAdvancesIndependentCounterWithoutTouchingRetryBudget() {
TaskFileJobEntity running = runningJob(111L, 1, LocalDateTime.now().minusHours(1));
running.setStuckRequeueCount(19);
TaskFileJobEntity pending = runningJob(111L, 1, LocalDateTime.now());
pending.setStatus("PENDING");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(pending));
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(1, result.resetCount());
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSet().contains("stuck_requeue_count"));
assertFalse(update.getValue().getSqlSet().contains("retry_count"),
"卡死重排仍不得消耗真失败预算");
assertTrue(update.getValue().getParamNameValuePairs().containsValue(20));
}
@Test
void stuckRequeueAtLimitFailsJobAndHandsItToTerminalCallback() {
TaskFileJobEntity running = runningJob(112L, 1, LocalDateTime.now().minusHours(1));
running.setStuckRequeueCount(TaskFileJobService.MAX_STUCK_REQUEUE);
TaskFileJobEntity failed = runningJob(112L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
failed.setStatus("FAILED");
failed.setErrorMessage("文件生成任务连续 20 次卡死重排,已判失败");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(failed));
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(0, result.resetCount(), "达到上限后不再重排");
assertEquals(List.of(failed), result.exhaustedJobs());
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getParamNameValuePairs().containsValue("FAILED"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(TaskFileJobService.MAX_RETRY_COUNT));
verify(applicationEventPublisher, never()).publishEvent(any());
}
@Test
void pendingFailureFinalizationIsReturnedAgainWithoutRequeueing() {
TaskFileJobEntity pending = runningJob(106L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
@@ -48,9 +48,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.after;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
@@ -349,6 +352,34 @@ class TaskResultFileJobWorkerStuckScanTest {
"只在应用就绪后执行: " + List.of(listener.value()));
}
@Test
void terminalParentHeartbeatStopsTouchingAndFailsJob() throws Exception {
// 父任务在 worker 执行中变终态:心跳不得再 touchRunning(否则 stuck 扫描永远看不到),
// 并主动把 job 收口为 FAILED(终态回调由 markFailedPermanent 打标)。
TaskResultFileJobWorker worker = buildWorker();
setLong(worker, "heartbeatIntervalMillis", 10L);
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(77L);
job.setTaskId(15L);
job.setModuleType("SIMILAR_ASIN");
job.setStatus("RUNNING");
when(taskFileJobService.isOwningTaskTerminal(15L)).thenReturn(true);
java.lang.reflect.Method method = TaskResultFileJobWorker.class
.getDeclaredMethod("startJobHeartbeat", TaskFileJobEntity.class);
method.setAccessible(true);
java.util.concurrent.ScheduledFuture<?> future =
(java.util.concurrent.ScheduledFuture<?>) method.invoke(worker, job);
try {
// heartbeat 间隔下限 1000ms,首轮在 1s 后才跑;给足调度余量
verify(taskFileJobService, timeout(3000))
.markFailedPermanent(eq(job), contains("父任务已终态"));
verify(taskFileJobService, after(150).never()).touchRunning(77L);
} finally {
future.cancel(true);
}
}
private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception {
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
field.setAccessible(true);
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.service.NodeSharedStore;
import com.nanri.aiimage.config.CostGuardProperties;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.notification.service.NotificationService;
@@ -12,6 +13,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.Duration;
import java.time.LocalDate;
import java.util.List;
@@ -41,6 +43,7 @@ class CostGuardSchedulerTest {
private DistributedJobLockService distributedJobLockService;
private CostGuardScheduler scheduler;
private CostGuardBaselineStore baselineStore;
@BeforeEach
void setUp() {
@@ -51,8 +54,16 @@ class CostGuardSchedulerTest {
properties.setDailyProxyThreshold(100);
properties.setDailyLlmCallsThreshold(100);
properties.setDailyLlmTokensThreshold(10_000);
scheduler = new CostGuardScheduler(properties, usageMapper, guardService,
sharedSecretMonitorService, notificationDispatchService, distributedJobLockService);
// null Redis/ObjectMapper ⇒ 本地实现;两个调度器共享同一个 store 即可模拟跨实例
baselineStore = new CostGuardBaselineStore(new NodeSharedStore<>("test-cost-guard",
Duration.ofHours(1), CostGuardBaselineStore.Baseline.class, null, null, 0L));
scheduler = newScheduler(properties);
}
private CostGuardScheduler newScheduler(CostGuardProperties properties) {
return new CostGuardScheduler(properties, usageMapper, guardService,
sharedSecretMonitorService, notificationDispatchService, distributedJobLockService,
baselineStore);
}
@Test
@@ -74,6 +85,7 @@ class CostGuardSchedulerTest {
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
.thenReturn(List.of(row(7L, "proxy", 12, 0)));
when(guardService.isPaused(7L)).thenReturn(false);
when(guardService.pause(anyLong(), anyString())).thenReturn(true);
scheduler.runOnce();
scheduler.runOnce();
@@ -104,6 +116,9 @@ class CostGuardSchedulerTest {
.thenReturn(List.of(row(9L, "proxy", 100, 0)))
.thenReturn(List.of(row(9L, "proxy", 101, 0)));
when(guardService.isPaused(9L)).thenReturn(false);
// 用户点继续时基线对齐到当前 100(不是删掉基线);否则首轮会按日累计立刻再暂停
when(usageMapper.sumForUser(eq(9L), any(LocalDate.class)))
.thenReturn(List.of(row(9L, "proxy", 100, 0)));
scheduler.runOnce();
scheduler.runOnce();
@@ -126,6 +141,95 @@ class CostGuardSchedulerTest {
verify(guardService, never()).pause(anyLong(), anyString());
}
@Test
void firstScanPausesWhenDailyAlreadyOverLimit() {
// 重启/首次部署后当天已经越线:首轮窗口不判,但日累计必须判,否则当天永远不会被拦
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 150, 0)));
when(guardService.isPaused(7L)).thenReturn(false);
when(guardService.pause(anyLong(), anyString())).thenReturn(true);
scheduler.runOnce();
verify(guardService).pause(eq(7L), contains("当日上限 100 次"));
verify(notificationDispatchService).pushToUser(eq(7L),
eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR),
anyString(), contains("继续任务"), isNull());
}
@Test
void pauseWriteFailureDoesNotNotify() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
.thenReturn(List.of(row(7L, "proxy", 12, 0)));
when(guardService.isPaused(7L)).thenReturn(false);
when(guardService.pause(anyLong(), anyString())).thenReturn(false);
scheduler.runOnce();
scheduler.runOnce();
verify(guardService).pause(eq(7L), contains("代理提取 12 次"));
verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(),
anyString(), anyString(), isNull());
}
@Test
void pauseWriteFailureRetriesNextScanInsteadOfSwallowingTheBreach() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
.thenReturn(List.of(row(7L, "proxy", 12, 0)))
.thenReturn(List.of(row(7L, "proxy", 13, 0)));
when(guardService.isPaused(7L)).thenReturn(false);
when(guardService.pause(anyLong(), anyString())).thenReturn(false, true);
scheduler.runOnce();
scheduler.runOnce();
scheduler.runOnce();
// 第一次写失败不推进基线,所以第三轮仍能看到增量并重试暂停
verify(guardService, org.mockito.Mockito.times(2))
.pause(eq(7L), contains("代理提取"));
verify(notificationDispatchService, org.mockito.Mockito.times(1)).pushToUser(eq(7L),
eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR),
anyString(), contains("继续任务"), isNull());
}
@Test
void multiMinuteGapIsNormalizedToWindowInsteadOfPausingFalsely() {
// 持锁方轮换:上一轮基线是 2 分钟前,增量 360 次。按 60 秒窗口归一化是 180 次,
// 不能把 360 当成一分钟增量(那会把正常的每分钟 180 次误判成失控)。
baselineStore.put(7L, new CostGuardBaselineStore.Baseline(0, 0, 0,
System.currentTimeMillis() - 120_000L));
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 360, 0)));
when(guardService.isPaused(7L)).thenReturn(false);
when(guardService.pause(anyLong(), anyString())).thenReturn(true);
scheduler.runOnce();
verify(guardService).pause(eq(7L), contains("代理提取 180 次"));
}
@Test
void resumeAlignsSharedBaselineSoAnotherInstanceDoesNotRepause() {
// 实例 A 写暂停并点继续;实例 B 接手后必须读到对齐后的基线,而不是上一轮的旧值
CostGuardProperties properties = new CostGuardProperties();
properties.setProxyWindowThreshold(10);
properties.setDailyProxyThreshold(100);
CostGuardScheduler instanceB = newScheduler(properties);
when(usageMapper.sumForUser(eq(9L), any(LocalDate.class)))
.thenReturn(List.of(row(9L, "proxy", 100, 0)));
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(9L, "proxy", 101, 0)));
when(guardService.isPaused(9L)).thenReturn(false);
scheduler.resetBaseline(9L);
instanceB.runOnce();
verify(guardService, never()).pause(anyLong(), anyString());
}
private UserSecretUsageAggregate row(long userId, String module, long calls, long tokens) {
UserSecretUsageAggregate row = new UserSecretUsageAggregate();
row.setUserId(userId);
@@ -78,6 +78,22 @@ class SharedSecretMonitorServiceTest {
anyString(), anyString(), anyString());
}
@Test
void nonHolderRefreshesSnapshotWithoutPushingDuplicateNotice() {
// 多实例下非持锁实例也要看到共享提示,但通知只由持锁方推,避免三台各推一遍
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
row(1L, "proxy", "same-cipher"),
row(2L, "proxy", "same-cipher"),
row(3L, "proxy", "same-cipher")));
service.refresh(false);
assertThat(service.warningsFor(1L)).hasSize(1);
assertThat(service.warningsFor(2L)).hasSize(1);
verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(),
anyString(), anyString(), anyString());
}
@Test
void llmModuleIsTypedAsLlm() {
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
@@ -40,7 +40,7 @@ class UserTaskGuardServiceTest {
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
when(guardMapper.selectById(7L)).thenReturn(null);
service.pause(7L, "LLM 消耗异常");
assertThat(service.pause(7L, "LLM 消耗异常")).isTrue();
verify(guardMapper).pause(7L, "LLM 消耗异常");
service.resume(7L);
@@ -56,6 +56,15 @@ class UserTaskGuardServiceTest {
assertThat(service.isPaused(0L)).isFalse();
}
@Test
void pauseWriteFailureReturnsFalseSoCallerDoesNotNotify() {
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
org.mockito.Mockito.doThrow(new IllegalStateException("db down"))
.when(guardMapper).pause(7L, "LLM 消耗异常");
assertThat(service.pause(7L, "LLM 消耗异常")).isFalse();
}
@Test
void mapperFailureFallsBackToNotPaused() {
when(guardMapper.selectById(7L)).thenThrow(new IllegalStateException("db down"));
@@ -79,6 +79,19 @@ class WorkbenchClientPackageServiceTest {
assertEquals("https://oss.aishufu.top/client/workbench-client/versions/put", result.get("upload_url"));
}
@Test
void presignKeysAreUniqueForTheSameVersion() {
// 同毫秒/同版本两次预签名必须不同:只靠时间戳会在 A/B 实例或并发上传时覆盖对象
when(ossStorageService.presignWorkbenchClientUpload(anyString(), eq(180))).thenReturn("put");
when(ossStorageService.getWorkbenchClientDownloadUrl(anyString())).thenReturn("download");
String first = (String) service().presignPackage("pkg.zip", "1.2.3").get("object_key");
String second = (String) service().presignPackage("pkg.zip", "1.2.3").get("object_key");
assertTrue(!first.equals(second), "同版本两次上传不得生成同一个对象 key");
assertTrue(first.endsWith("-1.2.3.zip") && second.endsWith("-1.2.3.zip"));
}
@Test
void presignRejectsNonZipAndBlankVersion() {
assertThrows(BusinessException.class, () -> service().presignPackage("pkg.exe", "1.0.0"));
@@ -108,6 +121,47 @@ class WorkbenchClientPackageServiceTest {
verify(packageMapper).insert(any(WorkbenchClientPackageEntity.class));
}
@Test
void confirmIsIdempotentForTheSameObjectKey() {
String objectKey = PREFIX + "20261024100000123-uuid-1.2.3.zip";
WorkbenchClientPackageEntity existing = row(99, "1.2.3", objectKey);
when(packageMapper.selectOne(any())).thenReturn(existing);
Map<String, Object> result = service().confirmPackage(
objectKey, "ShuFuAI-Workbench-1.2.3.zip", "1.2.3");
assertEquals(99L, result.get("id"));
verify(ossStorageService, never()).workbenchClientObjectSize(anyString());
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
}
@Test
void confirmRejectsEmptyObjectAndCleansItUp() {
String objectKey = PREFIX + "20261024100000123-uuid-1.2.3.zip";
when(ossStorageService.workbenchClientObjectSize(objectKey)).thenReturn(0L);
assertThrows(BusinessException.class, () -> service().confirmPackage(
objectKey, "ShuFuAI-Workbench-1.2.3.zip", "1.2.3"));
verify(ossStorageService).removeWorkbenchClientObject(objectKey);
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
}
@Test
void confirmRejectsDuplicateVersionToPreventSameVersionDifferentBytes() {
String objectKey = PREFIX + "20261024100000123-uuid-1.2.3.zip";
when(ossStorageService.workbenchClientObjectSize(objectKey)).thenReturn(1234L);
when(packageMapper.selectCount(any())).thenReturn(1L);
assertThrows(BusinessException.class, () -> service().confirmPackage(
objectKey, "ShuFuAI-Workbench-1.2.3.zip", "1.2.3"));
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
}
@Test
void confirmRejectsVersionThatDoesNotMatchObjectKey() {
String objectKey = PREFIX + "20261024100000123-1.2.3.zip";
@@ -23,6 +23,8 @@ export interface CostGuardStatus {
pausedAt: string | null
today: CostGuardUsage
window: CostGuardUsage & { seconds: number }
/** window 是本节点最近一轮扫描的读数,不是集群全局值。 */
windowLocalOnly?: boolean
thresholds: {
proxyWindow: number
llmCallsWindow: number
@@ -14,8 +14,9 @@ export function usageLine(status: CostGuardStatus | null): string {
if (!status) return ''
const today = status.today
const window = status.window
const scope = status.windowLocalOnly ? '(本节点读数)' : ''
return `今日代理提取 ${formatCount(today.proxy)} 次 · LLM ${formatCount(today.llmCalls)} 次 / ${formatCount(today.llmTokens)} tokens;`
+ `最近 ${window.seconds} 秒:代理 ${formatCount(window.proxy)} 次 · LLM ${formatCount(window.llmCalls)} 次 / ${formatCount(window.llmTokens)} tokens`
+ `最近 ${window.seconds} 秒${scope}:代理 ${formatCount(window.proxy)} 次 · LLM ${formatCount(window.llmCalls)} 次 / ${formatCount(window.llmTokens)} tokens`
}
/** 同一个暂停事件只弹一次:pausedAt 变化(新的自动暂停)才重新弹。 */