feat(cost-guard): 代理/LLM 损耗监控与账号级任务暂停
- 每分钟扫描用量增量/日累计,异常置暂停闸并推站内通知 - 品牌补测、跟价派发、外观激活、品牌检测与两个 LLM 客户端统一接闸 - 同一代理/LLM 密钥被 3 个以上账号共用时给出用户提示 - 品牌检测出口拒绝批级熔断;LLM 同 key 共享并发/间隔/429 惩罚 - V170 新增 biz_user_task_guard
This commit is contained in:
+20
-1
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmKeyGate;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
@@ -156,6 +157,10 @@ public class AppearancePatentLlmClient {
|
||||
private final ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final BrandCheckClient brandCheckClient;
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
|
||||
/** 损耗闸:字段注入避免改动手工构造的 LLM 用例;null=未启用(单测)。 */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
|
||||
private final UnreachableCooldown unreachableCooldown;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
@@ -656,11 +661,21 @@ public class AppearancePatentLlmClient {
|
||||
Exception lastFailure = null;
|
||||
AtomicInteger connectFailures = new AtomicInteger();
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire(apiKey)) {
|
||||
String content = invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
return content;
|
||||
} catch (Exception ex) {
|
||||
if (ex instanceof com.nanri.aiimage.common.exception.TaskPausedException paused) {
|
||||
// 暂停是账号级结论,重试不会变好;已完成的行走正常落库
|
||||
throw paused;
|
||||
}
|
||||
if (ex instanceof LlmKeyGate.QueueTimeoutException) {
|
||||
throw (LlmKeyGate.QueueTimeoutException) ex;
|
||||
}
|
||||
if (LlmKeyGate.isRateLimited(ex)) {
|
||||
LlmKeyGate.penalizeRateLimited(apiKey, 2_000L);
|
||||
}
|
||||
lastFailure = ex;
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
|
||||
@@ -1351,6 +1366,10 @@ public class AppearancePatentLlmClient {
|
||||
if (context == null) {
|
||||
return;
|
||||
}
|
||||
if (taskGuard != null && taskGuard.isPaused(context.userId())) {
|
||||
throw new com.nanri.aiimage.common.exception.TaskPausedException(
|
||||
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE);
|
||||
}
|
||||
userSecretUsageService.record(context.userId(), UserSecretModule.APPEARANCE_PATENT.key(), 1);
|
||||
}
|
||||
|
||||
|
||||
+8
@@ -109,6 +109,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
private final UserApiSecretService userApiSecretService;
|
||||
|
||||
/** 损耗闸:字段注入避免改动手工构造器与 golden 用例;null=未启用(单测)。 */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
|
||||
|
||||
/**
|
||||
* 参数顺序=迁移前 {@code @RequiredArgsConstructor} 的字段声明顺序,一字未动;
|
||||
* 其中五枚(fileTaskMapper/fileResultMapper/taskDistributedLockService/
|
||||
@@ -196,6 +200,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
|
||||
@Transactional
|
||||
public void activateTask(Long taskId, Long userId) {
|
||||
if (taskGuard != null && taskGuard.isPaused(userId)) {
|
||||
throw new com.nanri.aiimage.common.exception.BusinessException(com.nanri.aiimage.modules
|
||||
.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE);
|
||||
}
|
||||
recordSupport().activateTask(taskId, userId);
|
||||
}
|
||||
|
||||
|
||||
+51
-2
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.brand.client;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
@@ -71,6 +72,16 @@ public class BrandCheckClient {
|
||||
private static final int CONNECT_FAILURE_EVIDENCE = 2;
|
||||
private static final String REASON_SERVICE_UNREACHABLE = "品牌检测服务不可达(连接未建立,非本行数据问题)";
|
||||
|
||||
/**
|
||||
* 代理出口被上游拒绝的证据条数:同一批里已有这么多个品牌在换过出口后仍被拒,
|
||||
* 后面的品牌不再发起首轮探针。2026-10-23 现网:16890 每个品牌最多换 2 个 IP,
|
||||
* Java 再按品牌重试 10 轮,一批几千品牌就是几千×2 次提取;这个计数器把
|
||||
* 「整条出口池在被打」的结论限制在前 2 个品牌,其余直接转补测。
|
||||
*/
|
||||
private static final int EGRESS_REJECTED_EVIDENCE = 2;
|
||||
private static final String REASON_EGRESS_REJECTED =
|
||||
"代理出口 IP 连续被商标库拒绝,本批已暂停换 IP";
|
||||
|
||||
/** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */
|
||||
public static final String NO_PROXY_REASON = "用户未配置代理";
|
||||
|
||||
@@ -78,6 +89,13 @@ public class BrandCheckClient {
|
||||
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserApiSecretService userApiSecretService;
|
||||
private final com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown;
|
||||
|
||||
/**
|
||||
* 损耗暂停闸用字段注入:本类在单测里被大量手工 new(4 参构造),把闸门塞进构造器
|
||||
* 会让每条用例都多传一个 null。生产由 Spring 注入,单测里为 null 时按未暂停处理。
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
|
||||
/** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */
|
||||
private final ThreadLocal<AtomicBoolean> rejectedSubmission = new ThreadLocal<>();
|
||||
private volatile RestClient sharedRestClient;
|
||||
@@ -260,6 +278,16 @@ public class BrandCheckClient {
|
||||
// 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到
|
||||
// 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。
|
||||
Map<String, BrandCheckOutcome> outcomes = new LinkedHashMap<>();
|
||||
if (taskGuard != null && taskGuard.isPaused(userId)) {
|
||||
// 损耗闸已落下:整批一个请求都不发,结论按查询失败留待补测;重试不会让暂停变好
|
||||
log.warn("[brand-check] 账号任务已暂停,整批不打上游 userId={} 品牌数={}",
|
||||
userId, brands.size());
|
||||
for (String brand : brands) {
|
||||
outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand),
|
||||
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE));
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
String proxyUrl = proxyUrlFor(userId);
|
||||
if (proxyUrl.isBlank()) {
|
||||
// 缺代理整批一个请求都不发:这是配置问题,十条重试阶梯在这里只会白烧,
|
||||
@@ -293,6 +321,8 @@ public class BrandCheckClient {
|
||||
// 连接层证据跨品牌累计:批内任一次成功响应清零(见 checkOneBrand)。
|
||||
// 一次批量一份,不跨批复用——另一批的 upstreamRejecting 也是这个理由。
|
||||
AtomicInteger connectFailures = new AtomicInteger();
|
||||
// 出口被上游拒绝的证据跨品牌累计:达到阈值后不再给后续品牌发首轮探针。
|
||||
AtomicInteger egressRejections = new AtomicInteger();
|
||||
for (String brand : brands) {
|
||||
AtomicBoolean started = new AtomicBoolean(false);
|
||||
AtomicBoolean rejected = new AtomicBoolean(false);
|
||||
@@ -303,7 +333,7 @@ public class BrandCheckClient {
|
||||
() -> {
|
||||
started.set(true);
|
||||
return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting,
|
||||
proxyUrl, userId, connectFailures);
|
||||
proxyUrl, userId, connectFailures, egressRejections);
|
||||
},
|
||||
checkExecutor);
|
||||
} finally {
|
||||
@@ -333,7 +363,8 @@ public class BrandCheckClient {
|
||||
|
||||
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos,
|
||||
AtomicBoolean upstreamRejecting, String proxyUrl, Long userId,
|
||||
AtomicInteger connectFailures) {
|
||||
AtomicInteger connectFailures,
|
||||
AtomicInteger egressRejections) {
|
||||
int attempts = Math.max(1, properties.getRetryTimes());
|
||||
BrandCheckResponse response = null;
|
||||
Exception lastFailure = null;
|
||||
@@ -352,6 +383,14 @@ public class BrandCheckClient {
|
||||
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage());
|
||||
break;
|
||||
}
|
||||
// 出口池已被上游连续拒绝:后面的品牌连首轮探针都不发。与 upstreamRejecting
|
||||
// 不同,这里允许在 attempt=1 就短路——前 2 个品牌已经各烧过出口 IP,再让
|
||||
// 每个品牌各打一次只是把同一笔失败摊成品牌数×2 的提取费。
|
||||
if (egressRejections.get() >= EGRESS_REJECTED_EVIDENCE) {
|
||||
log.warn("[brand-check] 代理出口已被连续拒绝 {} 次,本品牌不再换 IP brand={} attempt={}/{}",
|
||||
egressRejections.get(), brand, attempt, attempts);
|
||||
return queryFailedOutcome(brand, REASON_EGRESS_REJECTED);
|
||||
}
|
||||
// 同批已有品牌走完整个阶梯、每一轮都被判查询失败 ⇒ 上游在确定性拒绝,本品牌别再烧
|
||||
// 等待与整批预算。2026-10-19 现网:品牌库代理套餐过期 → 16890 全部直连 WIPO →
|
||||
// 整片 Forbidden,单品牌 10 轮含递增等待约 45~65 秒全是白烧,而它烧掉的正是后面
|
||||
@@ -411,6 +450,16 @@ public class BrandCheckClient {
|
||||
if (response.getReason() != null && !response.getReason().isBlank()) {
|
||||
reason = response.getReason().trim();
|
||||
}
|
||||
if (BrandCheckFailureNotice.isEgressRejected(reason)) {
|
||||
// 这个品牌已经换过出口仍被拒:停止它自己的阶梯,并累计批级证据。
|
||||
// 16890 侧也有出口熔断,两层都只省“继续换 IP”的钱,不改“这行没查成”的结论。
|
||||
ladderCompleted = false;
|
||||
upstreamRejecting.set(true);
|
||||
int seen = egressRejections.incrementAndGet();
|
||||
log.warn("[brand-check] 代理出口被上游拒绝(本批第 {} 次),停止换 IP brand={} attempt={}/{} reason={}",
|
||||
seen, brand, attempt, attempts, reason);
|
||||
break;
|
||||
}
|
||||
if (response != null && !response.isRetryable()) {
|
||||
// 上游当场判定"重试也不会变好"(供应商拒付、套餐过期、这类结论是确定的),
|
||||
// 阶梯到此为止并把同批一起短路——旧版这里要跑满 10 轮、约 45~65 秒才知道
|
||||
|
||||
+6
@@ -70,6 +70,12 @@ public final class BrandCheckFailureNotice {
|
||||
return Kind.UPSTREAM;
|
||||
}
|
||||
|
||||
/** 原始原因是否属于「代理出口 IP 被上游拒绝」。批量熔断共用这一份判据。 */
|
||||
public static boolean isEgressRejected(String rawReason) {
|
||||
String reason = rawReason == null ? "" : rawReason.trim();
|
||||
return !reason.isEmpty() && EGRESS_REJECTED.matcher(reason).find();
|
||||
}
|
||||
|
||||
/** 用户可读的那一句:先说发生了什么,再说该做什么。 */
|
||||
public static String userMessage(Kind kind) {
|
||||
if (kind == null) {
|
||||
|
||||
+11
@@ -59,6 +59,10 @@ public class CollectDataBrandRecheckScheduler {
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
/** 损耗闸:字段注入避免改动手工构造的调度器用例;null=未启用(单测)。 */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.collect-data.brand-recheck.scan-interval-ms:300000}",
|
||||
initialDelayString = "${aiimage.collect-data.brand-recheck.scan-initial-delay-ms:120000}")
|
||||
public void scan() {
|
||||
@@ -124,6 +128,13 @@ public class CollectDataBrandRecheckScheduler {
|
||||
}
|
||||
return new GroupRound(0, 0);
|
||||
}
|
||||
if (taskGuard != null && taskGuard.isPaused(task.getUserId())) {
|
||||
// 暂停不消耗尝试次数:用户排查完点继续后,这批条目应按原退避节奏重新到期
|
||||
recheckService.postponeWithoutAttempt(entries, properties.getIntervalBaseMinutes());
|
||||
log.info("[brand-recheck] 账号任务已暂停,整组顺延 taskId={} userId={} 条目={}",
|
||||
taskId, task.getUserId(), entries.size());
|
||||
return new GroupRound(0, 0);
|
||||
}
|
||||
if (!STATUS_SUCCESS.equals(task.getStatus()) && !STATUS_FAILED.equals(task.getStatus())) {
|
||||
// 任务仍在跑:补测只做终态之后的兜底,顺延不计数(避免与分片重提竞态)
|
||||
recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes());
|
||||
|
||||
+6
@@ -60,6 +60,10 @@ public class NotificationService {
|
||||
public static final String SCENE_SERVICE_DOWN = "service_down";
|
||||
/** 麦象(18960 任务调度)异常:任务停滞/失败/队列积压。 */
|
||||
public static final String SCENE_MAIXIANG_ANOMALY = "maixiang_anomaly";
|
||||
/** 代理/LLM 消耗异常,账号任务已被自动暂停。 */
|
||||
public static final String SCENE_COST_GUARD = "cost_guard";
|
||||
/** 同一把代理/LLM 密钥被过多账号共用。 */
|
||||
public static final String SCENE_SECRET_SHARED = "secret_shared";
|
||||
public static final String SCENE_SYSTEM = "system";
|
||||
|
||||
/**
|
||||
@@ -78,10 +82,12 @@ public class NotificationService {
|
||||
private static final Map<String, String> SCENE_TO_CATEGORY = Map.of(
|
||||
SCENE_SERVICE_DOWN, CATEGORY_SYSTEM_ERROR,
|
||||
SCENE_MAIXIANG_ANOMALY, CATEGORY_SYSTEM_ERROR,
|
||||
SCENE_COST_GUARD, CATEGORY_SYSTEM_ERROR,
|
||||
SCENE_TASK_FAILED, CATEGORY_TASK_ERROR,
|
||||
SCENE_TASK_PARTIAL, CATEGORY_TASK_ERROR,
|
||||
SCENE_SECRET_BALANCE, CATEGORY_CONFIG_ERROR,
|
||||
SCENE_SECRET_INVALID, CATEGORY_CONFIG_ERROR,
|
||||
SCENE_SECRET_SHARED, CATEGORY_CONFIG_ERROR,
|
||||
SCENE_SYSTEM, CATEGORY_SYSTEM_NOTICE);
|
||||
|
||||
private static final long MAX_PAGE_SIZE = 100L;
|
||||
|
||||
+9
@@ -58,6 +58,10 @@ public class PriceTrackLoopRunService {
|
||||
private final PriceTrackLoopRunMapper loopRunMapper;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/** 损耗闸:字段注入避免改动手工 new 的 7 处测试构造器;null=未启用(单测)。 */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
|
||||
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
||||
|
||||
@Transactional
|
||||
@@ -104,6 +108,11 @@ public class PriceTrackLoopRunService {
|
||||
|
||||
@Transactional
|
||||
public PriceTrackLoopRunDispatchVo dispatchNext(Long loopRunId, Long userId) {
|
||||
if (taskGuard != null && taskGuard.isPaused(userId)) {
|
||||
// 不把循环标成 STOPPED:暂停只停派发,恢复后同一条循环还能接着跑
|
||||
throw new BusinessException(com.nanri.aiimage.modules.usersecret.service
|
||||
.UserTaskGuardService.PAUSED_MESSAGE);
|
||||
}
|
||||
PriceTrackLoopRunEntity entity = getOwnedLoopRun(loopRunId, userId);
|
||||
reconcileWithTerminalChild(entity);
|
||||
if (isTerminal(entity.getStatus())) {
|
||||
|
||||
+22
-1
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmKeyGate;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
@@ -50,6 +51,10 @@ public class SimilarAsinLlmClient {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
|
||||
/** 损耗闸:字段注入避免改动手工构造的 LLM 用例;null=未启用(单测)。 */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
|
||||
private final UnreachableCooldown unreachableCooldown;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
@@ -97,11 +102,23 @@ public class SimilarAsinLlmClient {
|
||||
Exception lastFailure = null;
|
||||
int connectFailures = 0;
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire(resolvedKey)) {
|
||||
String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
return content;
|
||||
} catch (Exception ex) {
|
||||
if (ex instanceof com.nanri.aiimage.common.exception.TaskPausedException paused) {
|
||||
// 暂停是账号级结论,重试不会变好;已完成的行走正常落库
|
||||
throw paused;
|
||||
}
|
||||
if (ex instanceof LlmKeyGate.QueueTimeoutException) {
|
||||
// 本地 key 容量已满:重试只会继续排队,直接交给行级失败
|
||||
throw (LlmKeyGate.QueueTimeoutException) ex;
|
||||
}
|
||||
if (LlmKeyGate.isRateLimited(ex)) {
|
||||
// 429 时把同 key 的下一次许可整体后移,避免所有调用一起再撞
|
||||
LlmKeyGate.penalizeRateLimited(resolvedKey, 2_000L);
|
||||
}
|
||||
lastFailure = ex;
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
connectFailures++;
|
||||
@@ -344,6 +361,10 @@ public class SimilarAsinLlmClient {
|
||||
if (context == null) {
|
||||
return;
|
||||
}
|
||||
if (taskGuard != null && taskGuard.isPaused(context.userId())) {
|
||||
throw new com.nanri.aiimage.common.exception.TaskPausedException(
|
||||
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE);
|
||||
}
|
||||
userSecretUsageService.record(context.userId(), UserSecretModule.SIMILAR_ASIN.key(), 1);
|
||||
}
|
||||
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.nanri.aiimage.modules.usersecret.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.usersecret.model.vo.CostGuardStatusVo;
|
||||
import com.nanri.aiimage.modules.usersecret.service.CostGuardService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* 桌面端损耗监控:身份一律取自 JWT,不接受前端传 uid。
|
||||
* 前端每 30 秒轮询 status;暂停时弹窗,用户排查后点「继续任务」调 resume。
|
||||
*/
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/user/cost-guard")
|
||||
@Tag(name = "损耗监控(桌面端)", description = "代理/LLM 异常消耗的暂停状态与手动继续。")
|
||||
public class UserCostGuardController {
|
||||
|
||||
private final CostGuardService costGuardService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@GetMapping("/status")
|
||||
@Operation(summary = "读取当前账号的损耗监控状态")
|
||||
public ApiResponse<CostGuardStatusVo> status(HttpServletRequest request) {
|
||||
return ApiResponse.success(costGuardService.status(currentUserId(request)));
|
||||
}
|
||||
|
||||
@PostMapping("/resume")
|
||||
@Operation(summary = "排查后继续任务", description = "只清暂停闸与损耗基线,不删除已完成任务和结果行。")
|
||||
public ApiResponse<CostGuardStatusVo> resume(HttpServletRequest request) {
|
||||
return ApiResponse.success("已继续任务", costGuardService.resume(currentUserId(request)));
|
||||
}
|
||||
|
||||
private Long currentUserId(HttpServletRequest request) {
|
||||
return adminAuthSupport.requireUser(request).getId();
|
||||
}
|
||||
}
|
||||
+32
@@ -1,11 +1,14 @@
|
||||
package com.nanri.aiimage.modules.usersecret.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate;
|
||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserSecretUsageEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface UserSecretUsageMapper extends BaseMapper<UserSecretUsageEntity> {
|
||||
@@ -33,6 +36,35 @@ public interface UserSecretUsageMapper extends BaseMapper<UserSecretUsageEntity>
|
||||
@Param("promptTokens") long promptTokens,
|
||||
@Param("completionTokens") long completionTokens);
|
||||
|
||||
/**
|
||||
* 当日用量按用户 × 模块聚合(损耗扫描每分钟读一次)。
|
||||
* 表是「用户 × 模块 × 天」的日聚合,行数 = 用户数 × 3,全量 GROUP BY 成本可忽略。
|
||||
*/
|
||||
@Select("""
|
||||
SELECT user_id AS userId,
|
||||
module_key AS moduleKey,
|
||||
COALESCE(SUM(call_count), 0) AS callCount,
|
||||
COALESCE(SUM(prompt_tokens + completion_tokens), 0) AS tokens
|
||||
FROM biz_user_secret_usage_daily
|
||||
WHERE business_date = #{businessDate}
|
||||
GROUP BY user_id, module_key
|
||||
""")
|
||||
List<UserSecretUsageAggregate> sumByUserAndModule(@Param("businessDate") LocalDate businessDate);
|
||||
|
||||
/** 单个用户的当日读数:客户端状态接口每 30 秒轮询一次,不能为此全表 GROUP BY。 */
|
||||
@Select("""
|
||||
SELECT user_id AS userId,
|
||||
module_key AS moduleKey,
|
||||
COALESCE(SUM(call_count), 0) AS callCount,
|
||||
COALESCE(SUM(prompt_tokens + completion_tokens), 0) AS tokens
|
||||
FROM biz_user_secret_usage_daily
|
||||
WHERE business_date = #{businessDate}
|
||||
AND user_id = #{userId}
|
||||
GROUP BY user_id, module_key
|
||||
""")
|
||||
List<UserSecretUsageAggregate> sumForUser(@Param("userId") Long userId,
|
||||
@Param("businessDate") LocalDate businessDate);
|
||||
|
||||
/** 分批删除保留期外的用量日统计行(该表此前只增不删)。 */
|
||||
int deleteBefore(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize);
|
||||
}
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.modules.usersecret.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
@Mapper
|
||||
public interface UserTaskGuardMapper extends BaseMapper<UserTaskGuardEntity> {
|
||||
|
||||
/** 幂等暂停:无记录则建,有记录则刷新原因与时刻。 */
|
||||
@Insert("""
|
||||
INSERT INTO biz_user_task_guard (user_id, paused, reason, paused_at, updated_at)
|
||||
VALUES (#{userId}, 1, #{reason}, NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE paused = 1, reason = #{reason},
|
||||
paused_at = NOW(), updated_at = NOW()
|
||||
""")
|
||||
int pause(@Param("userId") Long userId, @Param("reason") String reason);
|
||||
|
||||
@Update("""
|
||||
UPDATE biz_user_task_guard
|
||||
SET paused = 0, reason = '', paused_at = NULL, updated_at = NOW()
|
||||
WHERE user_id = #{userId}
|
||||
""")
|
||||
int resume(@Param("userId") Long userId);
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package com.nanri.aiimage.modules.usersecret.model.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/** 按用户 × 模块聚合的当日用量读数(CostGuardScheduler 的扫描输入)。 */
|
||||
@Data
|
||||
public class UserSecretUsageAggregate {
|
||||
|
||||
private Long userId;
|
||||
private String moduleKey;
|
||||
private Long callCount;
|
||||
private Long tokens;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.nanri.aiimage.modules.usersecret.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/** 账号级任务暂停闸:损耗异常时停新派发,排查后由用户手动恢复。 */
|
||||
@Data
|
||||
@TableName("biz_user_task_guard")
|
||||
public class UserTaskGuardEntity {
|
||||
|
||||
@TableId(value = "user_id", type = IdType.INPUT)
|
||||
private Long userId;
|
||||
private Boolean paused;
|
||||
private String reason;
|
||||
private LocalDateTime pausedAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.nanri.aiimage.modules.usersecret.model.vo;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/** 客户端「损耗监控」状态:暂停闸 + 当日用量 + 本轮增量 + 共享密钥提示。 */
|
||||
@Data
|
||||
public class CostGuardStatusVo {
|
||||
|
||||
private boolean paused;
|
||||
private String reason;
|
||||
private LocalDateTime pausedAt;
|
||||
private TodayUsage today;
|
||||
private WindowUsage window;
|
||||
private Thresholds thresholds;
|
||||
private List<SharedSecretWarningVo> sharedWarnings;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public static class TodayUsage {
|
||||
private long proxy;
|
||||
private long llmCalls;
|
||||
private long llmTokens;
|
||||
}
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public static class WindowUsage {
|
||||
private int seconds;
|
||||
private long proxy;
|
||||
private long llmCalls;
|
||||
private long llmTokens;
|
||||
}
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public static class Thresholds {
|
||||
private int proxyWindow;
|
||||
private int llmCallsWindow;
|
||||
private long llmTokensWindow;
|
||||
private int proxyDaily;
|
||||
private int llmCallsDaily;
|
||||
private long llmTokensDaily;
|
||||
private int sharedAccounts;
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.usersecret.model.vo;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
/** 一把代理/LLM 密钥被多个账号共用的提示(只报账号数,不暴露其他账号身份)。 */
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class SharedSecretWarningVo {
|
||||
|
||||
/** proxy / llm */
|
||||
private String type;
|
||||
private String moduleLabel;
|
||||
private int accounts;
|
||||
private int threshold;
|
||||
private String message;
|
||||
}
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.CostGuardProperties;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationService;
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate;
|
||||
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDate;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* 代理 / LLM 损耗扫描:每分钟读一次当日用量聚合表,用「本轮增量」判失控、用「日累计跨阈值」
|
||||
* 判慢烧;命中即置账号暂停闸并推用户通知。
|
||||
*
|
||||
* <p>为什么用日聚合表的增量而不是另建明细表:代理计次本来就是 16890/pt_worker 按批上报的,
|
||||
* LLM 计次在发请求前已落库;再建一条实时流水只会多一份可能对不上的账。日表行数 = 用户数×3,
|
||||
* 每分钟全量 GROUP BY 的成本可以忽略。
|
||||
*
|
||||
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
|
||||
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CostGuardScheduler {
|
||||
|
||||
static final String LOCK_NAME = "cost-guard-scan";
|
||||
|
||||
private final CostGuardProperties properties;
|
||||
private final UserSecretUsageMapper usageMapper;
|
||||
private final UserTaskGuardService guardService;
|
||||
private final SharedSecretMonitorService sharedSecretMonitorService;
|
||||
private final NotificationDispatchService notificationDispatchService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
|
||||
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
|
||||
private volatile long lastSharedRefreshAt = 0L;
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
|
||||
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
|
||||
public void scan() {
|
||||
if (!properties.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
|
||||
if (lock == null) {
|
||||
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
|
||||
return;
|
||||
}
|
||||
try (lock) {
|
||||
runOnce();
|
||||
} catch (Exception ex) {
|
||||
log.warn("[cost-guard] 损耗扫描异常终止 err={}", ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
|
||||
void runOnce() {
|
||||
Map<Long, UserTotals> current = loadTodayTotals();
|
||||
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
|
||||
int paused = 0;
|
||||
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
|
||||
Long userId = entry.getKey();
|
||||
UserTotals now = entry.getValue();
|
||||
UserTotals before = previousTotals.get(userId);
|
||||
if (before == null) {
|
||||
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
|
||||
deltas.put(userId, WindowDelta.ZERO);
|
||||
continue;
|
||||
}
|
||||
WindowDelta delta = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
deltas.put(userId, delta);
|
||||
if (guardService.isPaused(userId)) {
|
||||
continue;
|
||||
}
|
||||
String reason = breachReason(now, before, delta);
|
||||
if (reason == null) {
|
||||
continue;
|
||||
}
|
||||
guardService.pause(userId, reason);
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
paused++;
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
}
|
||||
previousTotals.clear();
|
||||
previousTotals.putAll(current);
|
||||
lastDeltas.clear();
|
||||
lastDeltas.putAll(deltas);
|
||||
maybeRefreshShared();
|
||||
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
|
||||
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
|
||||
}
|
||||
|
||||
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
|
||||
public void resetBaseline(Long userId) {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
previousTotals.remove(userId);
|
||||
lastDeltas.remove(userId);
|
||||
}
|
||||
|
||||
public WindowDelta lastDelta(Long userId) {
|
||||
WindowDelta delta = userId == null ? null : lastDeltas.get(userId);
|
||||
return delta == null ? WindowDelta.ZERO : delta;
|
||||
}
|
||||
|
||||
public Map<Long, UserTotals> loadTodayTotals() {
|
||||
List<UserSecretUsageAggregate> rows = usageMapper.sumByUserAndModule(LocalDate.now());
|
||||
Map<Long, UserTotals> totals = new LinkedHashMap<>();
|
||||
for (UserSecretUsageAggregate row : rows) {
|
||||
if (row.getUserId() == null || row.getUserId() <= 0) {
|
||||
continue;
|
||||
}
|
||||
accumulate(totals.computeIfAbsent(row.getUserId(), key -> new UserTotals()), row);
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
private void accumulate(UserTotals item, UserSecretUsageAggregate row) {
|
||||
long calls = row.getCallCount() == null ? 0L : row.getCallCount();
|
||||
long tokens = row.getTokens() == null ? 0L : row.getTokens();
|
||||
if (UserSecretModule.PROXY.key().equals(row.getModuleKey())) {
|
||||
item.proxy += calls;
|
||||
} else {
|
||||
item.llmCalls += calls;
|
||||
item.llmTokens += tokens;
|
||||
}
|
||||
}
|
||||
|
||||
public UserTotals todayTotals(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return new UserTotals();
|
||||
}
|
||||
try {
|
||||
UserTotals totals = new UserTotals();
|
||||
for (UserSecretUsageAggregate row : usageMapper.sumForUser(userId, LocalDate.now())) {
|
||||
accumulate(totals, row);
|
||||
}
|
||||
return totals;
|
||||
} catch (Exception ex) {
|
||||
// 状态接口是客户端每 30 秒一次的轮询:读不到就返回零值,不让一次库抖动打断提示
|
||||
log.warn("[cost-guard] 读取用户当日用量失败 userId={} err={}", userId, ex.getMessage());
|
||||
return new UserTotals();
|
||||
}
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
|
||||
int seconds = Math.max(60, properties.getWindowSeconds());
|
||||
if (properties.getProxyWindowThreshold() > 0
|
||||
&& delta.proxy >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
|
||||
+ properties.getProxyWindowThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowCallsThreshold() > 0
|
||||
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
|
||||
+ properties.getLlmWindowCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowTokensThreshold() > 0
|
||||
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
|
||||
+ properties.getLlmWindowTokensThreshold();
|
||||
}
|
||||
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
|
||||
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
|
||||
+ properties.getDailyProxyThreshold() + " 次";
|
||||
}
|
||||
if (crossed(before.llmCalls, now.llmCalls, properties.getDailyLlmCallsThreshold())) {
|
||||
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
|
||||
+ properties.getDailyLlmCallsThreshold() + " 次";
|
||||
}
|
||||
if (crossed(before.llmTokens, now.llmTokens, properties.getDailyLlmTokensThreshold())) {
|
||||
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
|
||||
+ properties.getDailyLlmTokensThreshold();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean crossed(long before, long now, long threshold) {
|
||||
return threshold > 0 && before < threshold && now >= threshold;
|
||||
}
|
||||
|
||||
private void maybeRefreshShared() {
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
|
||||
return;
|
||||
}
|
||||
lastSharedRefreshAt = now;
|
||||
try {
|
||||
sharedSecretMonitorService.refresh();
|
||||
} catch (Exception ex) {
|
||||
// 共享密钥扫描失败不影响损耗暂停主流程,下一轮再试
|
||||
log.warn("[cost-guard] 共享密钥扫描失败 err={}", ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class UserTotals {
|
||||
public long proxy;
|
||||
public long llmCalls;
|
||||
public long llmTokens;
|
||||
}
|
||||
|
||||
public record WindowDelta(long proxy, long llmCalls, long llmTokens) {
|
||||
public static final WindowDelta ZERO = new WindowDelta(0, 0, 0);
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.config.CostGuardProperties;
|
||||
import com.nanri.aiimage.modules.usersecret.model.vo.CostGuardStatusVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** 客户端损耗监控的状态读取与手动继续;不改任何任务结果行。 */
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CostGuardService {
|
||||
|
||||
private final CostGuardProperties properties;
|
||||
private final UserTaskGuardService guardService;
|
||||
private final CostGuardScheduler costGuardScheduler;
|
||||
private final SharedSecretMonitorService sharedSecretMonitorService;
|
||||
|
||||
public CostGuardStatusVo status(Long userId) {
|
||||
UserTaskGuardService.GuardState state = guardService.state(userId);
|
||||
CostGuardScheduler.UserTotals today = costGuardScheduler.todayTotals(userId);
|
||||
CostGuardScheduler.WindowDelta delta = costGuardScheduler.lastDelta(userId);
|
||||
int windowSeconds = Math.max(60, properties.getWindowSeconds());
|
||||
|
||||
CostGuardStatusVo vo = new CostGuardStatusVo();
|
||||
vo.setPaused(state.paused());
|
||||
vo.setReason(state.reason());
|
||||
vo.setPausedAt(state.pausedAt());
|
||||
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
|
||||
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
|
||||
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
|
||||
vo.setThresholds(new CostGuardStatusVo.Thresholds(
|
||||
properties.getProxyWindowThreshold(),
|
||||
properties.getLlmWindowCallsThreshold(),
|
||||
properties.getLlmWindowTokensThreshold(),
|
||||
properties.getDailyProxyThreshold(),
|
||||
properties.getDailyLlmCallsThreshold(),
|
||||
properties.getDailyLlmTokensThreshold(),
|
||||
properties.getSharedAccountsThreshold()));
|
||||
List<com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo> shared =
|
||||
sharedSecretMonitorService.warningsFor(userId);
|
||||
vo.setSharedWarnings(shared == null ? List.of() : shared);
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 用户排查后手动继续:清暂停位并把损耗基线对齐到当前,已完成的任务与结果原样保留。 */
|
||||
public CostGuardStatusVo resume(Long userId) {
|
||||
guardService.resume(userId);
|
||||
costGuardScheduler.resetBaseline(userId);
|
||||
return status(userId);
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.nanri.aiimage.config.CostGuardProperties;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationService;
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity;
|
||||
import com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo;
|
||||
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* 共享密钥提示:同一把代理/LLM 密钥被过多账号共用时,给每个使用者一条明确提示。
|
||||
*
|
||||
* <p>只统计、不暂停:共用同一把密钥本身不是故障(账单按 key 计费,平台也允许这样配),
|
||||
* 但它会把一次异常消耗放大到所有账号,也会让「谁烧的」无法归因。提示里只报账号数,
|
||||
* 不暴露其他账号身份。
|
||||
*
|
||||
* <p>分组键直接用密文:{@code ShopCredentialCryptoService} 是 AES/ECB 无 IV,
|
||||
* 同明文必同密文(与 {@link UserApiSecretService#checkAllForScheduler} 同一口径),
|
||||
* 因此本类不需要解密、也不会把明文密钥带进内存快照。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class SharedSecretMonitorService {
|
||||
|
||||
private final UserApiSecretMapper userApiSecretMapper;
|
||||
private final CostGuardProperties properties;
|
||||
private final NotificationDispatchService notificationDispatchService;
|
||||
|
||||
private volatile Map<Long, List<SharedSecretWarningVo>> warningsByUser = Map.of();
|
||||
private volatile long snapshotAtMillis = 0L;
|
||||
|
||||
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
|
||||
public synchronized void refresh() {
|
||||
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
|
||||
warningsByUser = Map.of();
|
||||
snapshotAtMillis = System.currentTimeMillis();
|
||||
return;
|
||||
}
|
||||
// 用字符串列名而不是 lambda:本类在纯单测里不经过 MyBatis 的实体元数据初始化
|
||||
List<UserApiSecretEntity> rows = userApiSecretMapper.selectList(
|
||||
new QueryWrapper<UserApiSecretEntity>()
|
||||
.select("user_id", "module_key", "secret_value"));
|
||||
// groupKey = module|密文;值是该组的使用账号集合(去重)
|
||||
Map<String, SecretGroup> groups = new LinkedHashMap<>();
|
||||
for (UserApiSecretEntity row : rows) {
|
||||
if (row.getUserId() == null || row.getUserId() <= 0) {
|
||||
continue;
|
||||
}
|
||||
Optional<UserSecretModule> module = UserSecretModule.of(row.getModuleKey());
|
||||
if (module.isEmpty() || row.getSecretValue() == null || row.getSecretValue().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
String groupKey = row.getModuleKey() + '|' + row.getSecretValue();
|
||||
groups.computeIfAbsent(groupKey, key -> new SecretGroup(module.get()))
|
||||
.userIds.add(row.getUserId());
|
||||
}
|
||||
|
||||
Map<Long, List<SharedSecretWarningVo>> next = new LinkedHashMap<>();
|
||||
int hitGroups = 0;
|
||||
for (Map.Entry<String, SecretGroup> entry : groups.entrySet()) {
|
||||
SecretGroup group = entry.getValue();
|
||||
int accounts = group.userIds.size();
|
||||
if (accounts < properties.getSharedAccountsThreshold()) {
|
||||
continue;
|
||||
}
|
||||
hitGroups++;
|
||||
String type = group.module == UserSecretModule.PROXY ? "proxy" : "llm";
|
||||
int threshold = properties.getSharedAccountsThreshold();
|
||||
String message = buildMessage(group.module, accounts, threshold);
|
||||
SharedSecretWarningVo warning = new SharedSecretWarningVo(
|
||||
type, group.module.label(), accounts, threshold, message);
|
||||
for (Long userId : group.userIds) {
|
||||
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
}
|
||||
}
|
||||
warningsByUser = next;
|
||||
snapshotAtMillis = System.currentTimeMillis();
|
||||
log.info("[cost-guard] 共享密钥扫描完成 配置行={} 密钥组={} 超阈值组={} 受影响账号={}",
|
||||
rows.size(), groups.size(), hitGroups, next.size());
|
||||
}
|
||||
|
||||
/** 当前用户共享密钥提示;快照未建立时返回空,不阻塞客户端轮询。 */
|
||||
public List<SharedSecretWarningVo> warningsFor(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return List.of();
|
||||
}
|
||||
return warningsByUser.getOrDefault(userId, List.of());
|
||||
}
|
||||
|
||||
public long snapshotAtMillis() {
|
||||
return snapshotAtMillis;
|
||||
}
|
||||
|
||||
private void pushDailyWarning(Long userId, UserSecretModule module, String groupKey,
|
||||
SharedSecretWarningVo warning) {
|
||||
String day = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
|
||||
String dedupe = "secret_shared:" + module.key() + ':' + sha256Hex(groupKey)
|
||||
+ ':' + userId + ':' + day;
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_SECRET_SHARED,
|
||||
NotificationService.LEVEL_WARNING,
|
||||
module.label() + "被 " + warning.getAccounts() + " 个账号共用",
|
||||
warning.getMessage(), dedupe);
|
||||
}
|
||||
|
||||
private String buildMessage(UserSecretModule module, int accounts, int threshold) {
|
||||
String what = module == UserSecretModule.PROXY ? "代理提取链接" : "LLM 密钥";
|
||||
return "你的" + module.label() + "(" + what + ")当前被 " + accounts
|
||||
+ " 个账号共用,已达到提示阈值 " + threshold + "。任一账号的异常消耗都会算在这把密钥的套餐上,"
|
||||
+ "建议在「密钥设置」里为不同账号拆分独立密钥,便于控费和定位问题。";
|
||||
}
|
||||
|
||||
private String sha256Hex(String value) {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8)));
|
||||
} catch (Exception ex) {
|
||||
return Integer.toHexString(value.hashCode());
|
||||
}
|
||||
}
|
||||
|
||||
private static final class SecretGroup {
|
||||
private final UserSecretModule module;
|
||||
private final Set<Long> userIds = new HashSet<>();
|
||||
|
||||
private SecretGroup(UserSecretModule module) {
|
||||
this.module = module;
|
||||
}
|
||||
}
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserTaskGuardMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 账号级任务暂停闸:损耗异常时由 CostGuardScheduler 置位,派发入口据此停新任务。
|
||||
*
|
||||
* <p>读侧带 3 秒内存缓存:品牌检测/LLM 每次请求都问一次,直接打库会把闸门本身变成热点;
|
||||
* 多实例下最迟 3 秒收敛,足够挡住下一分钟的任务。写侧立即清缓存,本机无延迟。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class UserTaskGuardService {
|
||||
|
||||
public static final String PAUSED_MESSAGE =
|
||||
"账号任务已暂停(损耗保护),请排查代理/LLM 消耗后在客户端点击「继续任务」";
|
||||
|
||||
private static final long CACHE_TTL_NANOS = TimeUnit.SECONDS.toNanos(3);
|
||||
private static final int REASON_MAX_LENGTH = 500;
|
||||
|
||||
private final UserTaskGuardMapper guardMapper;
|
||||
private final ConcurrentHashMap<Long, CachedState> cache = new ConcurrentHashMap<>();
|
||||
|
||||
public boolean isPaused(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return false;
|
||||
}
|
||||
CachedState cached = cache.get(userId);
|
||||
if (cached != null && System.nanoTime() - cached.readAtNanos() < CACHE_TTL_NANOS) {
|
||||
return cached.paused();
|
||||
}
|
||||
GuardState state = load(userId);
|
||||
cache.put(userId, new CachedState(state.paused(), state.reason(), state.pausedAt(),
|
||||
System.nanoTime()));
|
||||
return state.paused();
|
||||
}
|
||||
|
||||
/** 状态查询走库:客户端弹窗要看到其他实例刚写入的暂停,不吃本机缓存。 */
|
||||
public GuardState state(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return GuardState.ACTIVE;
|
||||
}
|
||||
GuardState state = load(userId);
|
||||
cache.put(userId, new CachedState(state.paused(), state.reason(), state.pausedAt(),
|
||||
System.nanoTime()));
|
||||
return state;
|
||||
}
|
||||
|
||||
public void pause(Long userId, String reason) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return;
|
||||
}
|
||||
String safeReason = truncate(reason);
|
||||
try {
|
||||
guardMapper.pause(userId, safeReason);
|
||||
cache.remove(userId);
|
||||
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
|
||||
} catch (Exception ex) {
|
||||
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
|
||||
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
|
||||
userId, safeReason, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 用户手动继续:只清暂停位,不删任何已完成的任务/结果。 */
|
||||
public boolean resume(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
guardMapper.resume(userId);
|
||||
cache.remove(userId);
|
||||
log.info("[cost-guard] 用户手动继续任务 userId={}", userId);
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
log.error("[cost-guard] 继续任务写入失败 userId={} err={}", userId, ex.getMessage(), ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private GuardState load(Long userId) {
|
||||
try {
|
||||
UserTaskGuardEntity row = guardMapper.selectById(userId);
|
||||
if (row == null || !Boolean.TRUE.equals(row.getPaused())) {
|
||||
return GuardState.ACTIVE;
|
||||
}
|
||||
return new GuardState(true, row.getReason() == null ? "" : row.getReason(),
|
||||
row.getPausedAt());
|
||||
} catch (Exception ex) {
|
||||
// 读不到闸门状态时按「不暂停」走:数据库抖动不该把全体用户的合法任务一起冻住。
|
||||
// 真正的失控由下一轮扫描重新置位。
|
||||
log.warn("[cost-guard] 读取暂停状态失败,按未暂停处理 userId={} err={}",
|
||||
userId, ex.getMessage());
|
||||
return GuardState.ACTIVE;
|
||||
}
|
||||
}
|
||||
|
||||
private String truncate(String reason) {
|
||||
String value = reason == null ? "" : reason.trim();
|
||||
return value.length() <= REASON_MAX_LENGTH ? value : value.substring(0, REASON_MAX_LENGTH);
|
||||
}
|
||||
|
||||
private record CachedState(boolean paused, String reason, LocalDateTime pausedAt, long readAtNanos) {
|
||||
}
|
||||
|
||||
public record GuardState(boolean paused, String reason, LocalDateTime pausedAt) {
|
||||
public static final GuardState ACTIVE = new GuardState(false, "", null);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user