feat(cost-guard): 代理/LLM 损耗监控与账号级任务暂停

- 每分钟扫描用量增量/日累计,异常置暂停闸并推站内通知
- 品牌补测、跟价派发、外观激活、品牌检测与两个 LLM 客户端统一接闸
- 同一代理/LLM 密钥被 3 个以上账号共用时给出用户提示
- 品牌检测出口拒绝批级熔断;LLM 同 key 共享并发/间隔/429 惩罚
- V170 新增 biz_user_task_guard
This commit is contained in:
2026-10-24 10:50:20 +08:00
parent 052f2bd4a5
commit 9c4905c94e
32 changed files with 1625 additions and 5 deletions
@@ -0,0 +1,12 @@
package com.nanri.aiimage.common.exception;
/**
* 账号任务已被损耗保护暂停:调用方必须立即停止重试(这不是上游抖动,重试不会变好),
* 保留已完成数据,等用户在客户端点「继续任务」后再重跑剩余部分。
*/
public class TaskPausedException extends RuntimeException {
public TaskPausedException(String message) {
super(message);
}
}
@@ -0,0 +1,186 @@
package com.nanri.aiimage.common.retry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.client.HttpStatusCodeException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* 同一把 LLM key 的进程内全局请求门:并发上限 + 最小间隔 + 429 共享冷却。
*
* <p>多个账号配同一把 key 时,provider 的 RPM/TPM 是共享的;如果每个调用各自
* 重试,429 会互相放大。这里把并发和节奏收到 key 级别,让排队替代重试:
* 正常时几乎不等待,打满时后来者按顺序等,避免全员 429。
*
* <p>配置(环境变量,默认保守;按 provider 实际额度调):
* <ul>
* <li>{@code AIIMAGE_LLM_KEY_MAX_CONCURRENCY}:同 key 最大在途数,默认 8</li>
* <li>{@code AIIMAGE_LLM_KEY_MIN_INTERVAL_MS}:同 key 两次请求最小间隔,默认 50ms</li>
* <li>{@code AIIMAGE_LLM_KEY_ACQUIRE_TIMEOUT_MS}:排队等待上限,默认 120000ms</li>
* </ul>
*
* <p>只存 key 的 SHA-256 指纹,不把原文写进 map/日志。
*/
@Slf4j
public final class LlmKeyGate {
private static final int MAX_CONCURRENCY = positiveInt("AIIMAGE_LLM_KEY_MAX_CONCURRENCY", 8);
private static final long MIN_INTERVAL_NANOS =
TimeUnit.MILLISECONDS.toNanos(positiveLong("AIIMAGE_LLM_KEY_MIN_INTERVAL_MS", 50L));
private static final long ACQUIRE_TIMEOUT_NANOS =
TimeUnit.MILLISECONDS.toNanos(positiveLong("AIIMAGE_LLM_KEY_ACQUIRE_TIMEOUT_MS", 120_000L));
private static final ConcurrentHashMap<String, Gate> GATES = new ConcurrentHashMap<>();
private LlmKeyGate() {
}
/** 同 key 的排队许可;close 必须调用(try-with-resources 自动归还)。 */
public static final class Permit implements AutoCloseable {
private final Gate gate;
private Permit(Gate gate) {
this.gate = gate;
}
@Override
public void close() {
gate.release();
}
}
/** 排队超时:这是本地容量结论,重试只会继续排队,调用方应停止阶梯。 */
public static final class QueueTimeoutException extends IllegalStateException {
public QueueTimeoutException(String message) {
super(message);
}
}
public static Permit acquire(String apiKey) {
Gate gate = GATES.computeIfAbsent(fingerprint(apiKey), ignored -> new Gate());
return gate.acquire();
}
/** 429 时把同 key 的下一次许可整体后移,避免所有线程同时再撞一次。 */
public static void penalizeRateLimited(String apiKey, long millis) {
GATES.computeIfAbsent(fingerprint(apiKey), ignored -> new Gate())
.penalize(TimeUnit.MILLISECONDS.toNanos(Math.max(0L, millis)));
}
/** 从异常链里识别 HTTP 429;其他异常(超时/5xx)不改变 key 级节奏。 */
public static boolean isRateLimited(Throwable error) {
Throwable current = error;
while (current != null) {
if (current instanceof HttpStatusCodeException status
&& status.getStatusCode().value() == 429) {
return true;
}
current = current.getCause();
}
return false;
}
public static void resetForTests() {
GATES.clear();
}
private static String fingerprint(String apiKey) {
String key = apiKey == null ? "" : apiKey.trim();
try {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(key.getBytes(StandardCharsets.UTF_8));
StringBuilder hex = new StringBuilder(digest.length * 2);
for (byte b : digest) {
hex.append(String.format("%02x", b));
}
return hex.toString();
} catch (NoSuchAlgorithmException ex) {
// SHA-256 是 JDK 必备算法;真缺失时退化为长度+哈希码,仍不落原文
return "fallback:" + key.length() + ":" + key.hashCode();
}
}
private static int positiveInt(String name, int fallback) {
try {
int value = Integer.parseInt(System.getenv().getOrDefault(name, "").trim());
return value > 0 ? value : fallback;
} catch (RuntimeException ex) {
return fallback;
}
}
private static long positiveLong(String name, long fallback) {
try {
long value = Long.parseLong(System.getenv().getOrDefault(name, "").trim());
return value > 0 ? value : fallback;
} catch (RuntimeException ex) {
return fallback;
}
}
private static final class Gate {
private final Semaphore permits = new Semaphore(MAX_CONCURRENCY);
private final Object lock = new Object();
private final AtomicLong nextAtNanos = new AtomicLong();
Permit acquire() {
try {
if (!permits.tryAcquire(ACQUIRE_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) {
throw new QueueTimeoutException(
"LLM key 排队超时(同 key 并发已满,等待上限 "
+ TimeUnit.NANOSECONDS.toMillis(ACQUIRE_TIMEOUT_NANOS) + "ms)");
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException("LLM key 排队被中断", ex);
}
try {
waitForInterval();
return new Permit(this);
} catch (RuntimeException ex) {
permits.release();
throw ex;
}
}
void release() {
permits.release();
}
void penalize(long nanos) {
if (nanos <= 0) {
return;
}
long now = System.nanoTime();
nextAtNanos.accumulateAndGet(now + nanos, Math::max);
}
private void waitForInterval() {
long interval = MIN_INTERVAL_NANOS;
while (interval > 0) {
long now = System.nanoTime();
long next = nextAtNanos.get();
long wait = next - now;
if (wait <= 0) {
long assigned = now + interval;
if (nextAtNanos.compareAndSet(next, assigned)) {
return;
}
continue;
}
try {
TimeUnit.NANOSECONDS.sleep(Math.min(wait, TimeUnit.MILLISECONDS.toNanos(50)));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException("LLM key 间隔等待被中断", ex);
}
}
}
}
}
@@ -0,0 +1,52 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* 代理 / LLM 损耗保护:按分钟增量与当日累计两条线判定异常,命中即暂停该账号的新任务。
*
* <p>阈值默认值按「正常批次不该碰、失控循环一定碰」取:品牌补测单批最多百来个品牌,
* 一分钟内正常提取几十次;失控时一个账号一分钟能烧到几百上千次。上线后按后台用量
* 曲线调整,不把阈值写死在代码里。
*/
@Data
@ConfigurationProperties(prefix = "aiimage.cost-guard")
public class CostGuardProperties {
/** 总开关:关掉只停自动暂停与提示,记账与状态接口仍可用。 */
private boolean enabled = true;
/** 扫描间隔(毫秒);用量是按分钟增量判定的,默认 60 秒。 */
private long scanIntervalMs = 60_000L;
/** 首次扫描延迟(毫秒):避开应用启动时的迁移与预热。 */
private long scanInitialDelayMs = 30_000L;
/** 展示给用户的窗口长度(秒);扫描间隔默认 60 秒,含义是「最近约一分钟增量」。 */
private int windowSeconds = 60;
/** 窗口内代理提取次数阈值;超过即暂停。 */
private int proxyWindowThreshold = 600;
/** 窗口内 LLM 调用次数阈值。 */
private int llmWindowCallsThreshold = 180;
/** 窗口内 LLM token 阈值(prompt + completion)。 */
private long llmWindowTokensThreshold = 600_000L;
/** 当日代理提取次数上限;0=关闭。 */
private int dailyProxyThreshold = 8_000;
/** 当日 LLM 调用次数上限;0=关闭。 */
private int dailyLlmCallsThreshold = 6_000;
/** 当日 LLM token 上限;0=关闭。 */
private long dailyLlmTokensThreshold = 20_000_000L;
/** 同一把代理/LLM 密钥被多少个账号共用时提示;0=关闭。 */
private int sharedAccountsThreshold = 3;
/** 共享密钥扫描间隔(毫秒);首轮随第一次损耗扫描一起跑。 */
private long sharedScanIntervalMs = 600_000L;
}
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class})
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class, CostGuardProperties.class})
public class PropertiesConfig {
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmKeyGate;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
import com.nanri.aiimage.config.AppearancePatentProperties;
@@ -156,6 +157,10 @@ public class AppearancePatentLlmClient {
private final ExternalCallMetricsRecorder externalCallMetrics;
private final BrandCheckClient brandCheckClient;
private final UserSecretUsageService userSecretUsageService;
/** 损耗闸:字段注入避免改动手工构造的 LLM 用例;null=未启用(单测)。 */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
private final UnreachableCooldown unreachableCooldown;
private volatile RestClient sharedRestClient;
@@ -656,11 +661,21 @@ public class AppearancePatentLlmClient {
Exception lastFailure = null;
AtomicInteger connectFailures = new AtomicInteger();
for (int attempt = 1; attempt <= attempts; attempt++) {
try {
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire(apiKey)) {
String content = invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
unreachableCooldown.recordConnected(host);
return content;
} catch (Exception ex) {
if (ex instanceof com.nanri.aiimage.common.exception.TaskPausedException paused) {
// 暂停是账号级结论,重试不会变好;已完成的行走正常落库
throw paused;
}
if (ex instanceof LlmKeyGate.QueueTimeoutException) {
throw (LlmKeyGate.QueueTimeoutException) ex;
}
if (LlmKeyGate.isRateLimited(ex)) {
LlmKeyGate.penalizeRateLimited(apiKey, 2_000L);
}
lastFailure = ex;
if (UnreachableCooldown.isConnectFailure(ex)) {
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
@@ -1351,6 +1366,10 @@ public class AppearancePatentLlmClient {
if (context == null) {
return;
}
if (taskGuard != null && taskGuard.isPaused(context.userId())) {
throw new com.nanri.aiimage.common.exception.TaskPausedException(
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE);
}
userSecretUsageService.record(context.userId(), UserSecretModule.APPEARANCE_PATENT.key(), 1);
}
@@ -109,6 +109,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
private final TaskProgressLightAssembler taskProgressLightAssembler;
private final UserApiSecretService userApiSecretService;
/** 损耗闸:字段注入避免改动手工构造器与 golden 用例;null=未启用(单测)。 */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
/**
* 参数顺序=迁移前 {@code @RequiredArgsConstructor} 的字段声明顺序,一字未动;
* 其中五枚(fileTaskMapper/fileResultMapper/taskDistributedLockService/
@@ -196,6 +200,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
@Transactional
public void activateTask(Long taskId, Long userId) {
if (taskGuard != null && taskGuard.isPaused(userId)) {
throw new com.nanri.aiimage.common.exception.BusinessException(com.nanri.aiimage.modules
.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE);
}
recordSupport().activateTask(taskId, userId);
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.brand.client;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.nanri.aiimage.config.BrandCheckProperties;
import com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
import jakarta.annotation.PreDestroy;
@@ -71,6 +72,16 @@ public class BrandCheckClient {
private static final int CONNECT_FAILURE_EVIDENCE = 2;
private static final String REASON_SERVICE_UNREACHABLE = "品牌检测服务不可达(连接未建立,非本行数据问题)";
/**
* 代理出口被上游拒绝的证据条数:同一批里已有这么多个品牌在换过出口后仍被拒,
* 后面的品牌不再发起首轮探针。2026-10-23 现网:16890 每个品牌最多换 2 个 IP,
* Java 再按品牌重试 10 轮,一批几千品牌就是几千×2 次提取;这个计数器把
* 「整条出口池在被打」的结论限制在前 2 个品牌,其余直接转补测。
*/
private static final int EGRESS_REJECTED_EVIDENCE = 2;
private static final String REASON_EGRESS_REJECTED =
"代理出口 IP 连续被商标库拒绝,本批已暂停换 IP";
/** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */
public static final String NO_PROXY_REASON = "用户未配置代理";
@@ -78,6 +89,13 @@ public class BrandCheckClient {
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
private final UserApiSecretService userApiSecretService;
private final com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown;
/**
* 损耗暂停闸用字段注入:本类在单测里被大量手工 new(4 参构造),把闸门塞进构造器
* 会让每条用例都多传一个 null。生产由 Spring 注入,单测里为 null 时按未暂停处理。
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
/** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */
private final ThreadLocal<AtomicBoolean> rejectedSubmission = new ThreadLocal<>();
private volatile RestClient sharedRestClient;
@@ -260,6 +278,16 @@ public class BrandCheckClient {
// 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到
// 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。
Map<String, BrandCheckOutcome> outcomes = new LinkedHashMap<>();
if (taskGuard != null && taskGuard.isPaused(userId)) {
// 损耗闸已落下:整批一个请求都不发,结论按查询失败留待补测;重试不会让暂停变好
log.warn("[brand-check] 账号任务已暂停,整批不打上游 userId={} 品牌数={}",
userId, brands.size());
for (String brand : brands) {
outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand),
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE));
}
return outcomes;
}
String proxyUrl = proxyUrlFor(userId);
if (proxyUrl.isBlank()) {
// 缺代理整批一个请求都不发:这是配置问题,十条重试阶梯在这里只会白烧,
@@ -293,6 +321,8 @@ public class BrandCheckClient {
// 连接层证据跨品牌累计:批内任一次成功响应清零(见 checkOneBrand)。
// 一次批量一份,不跨批复用——另一批的 upstreamRejecting 也是这个理由。
AtomicInteger connectFailures = new AtomicInteger();
// 出口被上游拒绝的证据跨品牌累计:达到阈值后不再给后续品牌发首轮探针。
AtomicInteger egressRejections = new AtomicInteger();
for (String brand : brands) {
AtomicBoolean started = new AtomicBoolean(false);
AtomicBoolean rejected = new AtomicBoolean(false);
@@ -303,7 +333,7 @@ public class BrandCheckClient {
() -> {
started.set(true);
return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting,
proxyUrl, userId, connectFailures);
proxyUrl, userId, connectFailures, egressRejections);
},
checkExecutor);
} finally {
@@ -333,7 +363,8 @@ public class BrandCheckClient {
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos,
AtomicBoolean upstreamRejecting, String proxyUrl, Long userId,
AtomicInteger connectFailures) {
AtomicInteger connectFailures,
AtomicInteger egressRejections) {
int attempts = Math.max(1, properties.getRetryTimes());
BrandCheckResponse response = null;
Exception lastFailure = null;
@@ -352,6 +383,14 @@ public class BrandCheckClient {
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage());
break;
}
// 出口池已被上游连续拒绝:后面的品牌连首轮探针都不发。与 upstreamRejecting
// 不同,这里允许在 attempt=1 就短路——前 2 个品牌已经各烧过出口 IP,再让
// 每个品牌各打一次只是把同一笔失败摊成品牌数×2 的提取费。
if (egressRejections.get() >= EGRESS_REJECTED_EVIDENCE) {
log.warn("[brand-check] 代理出口已被连续拒绝 {} 次,本品牌不再换 IP brand={} attempt={}/{}",
egressRejections.get(), brand, attempt, attempts);
return queryFailedOutcome(brand, REASON_EGRESS_REJECTED);
}
// 同批已有品牌走完整个阶梯、每一轮都被判查询失败 ⇒ 上游在确定性拒绝,本品牌别再烧
// 等待与整批预算。2026-10-19 现网:品牌库代理套餐过期 → 16890 全部直连 WIPO →
// 整片 Forbidden,单品牌 10 轮含递增等待约 45~65 秒全是白烧,而它烧掉的正是后面
@@ -411,6 +450,16 @@ public class BrandCheckClient {
if (response.getReason() != null && !response.getReason().isBlank()) {
reason = response.getReason().trim();
}
if (BrandCheckFailureNotice.isEgressRejected(reason)) {
// 这个品牌已经换过出口仍被拒:停止它自己的阶梯,并累计批级证据。
// 16890 侧也有出口熔断,两层都只省“继续换 IP”的钱,不改“这行没查成”的结论。
ladderCompleted = false;
upstreamRejecting.set(true);
int seen = egressRejections.incrementAndGet();
log.warn("[brand-check] 代理出口被上游拒绝(本批第 {} 次),停止换 IP brand={} attempt={}/{} reason={}",
seen, brand, attempt, attempts, reason);
break;
}
if (response != null && !response.isRetryable()) {
// 上游当场判定"重试也不会变好"(供应商拒付、套餐过期、这类结论是确定的),
// 阶梯到此为止并把同批一起短路——旧版这里要跑满 10 轮、约 45~65 秒才知道
@@ -70,6 +70,12 @@ public final class BrandCheckFailureNotice {
return Kind.UPSTREAM;
}
/** 原始原因是否属于「代理出口 IP 被上游拒绝」。批量熔断共用这一份判据。 */
public static boolean isEgressRejected(String rawReason) {
String reason = rawReason == null ? "" : rawReason.trim();
return !reason.isEmpty() && EGRESS_REJECTED.matcher(reason).find();
}
/** 用户可读的那一句:先说发生了什么,再说该做什么。 */
public static String userMessage(Kind kind) {
if (kind == null) {
@@ -59,6 +59,10 @@ public class CollectDataBrandRecheckScheduler {
private final TaskFileJobService taskFileJobService;
private final DistributedJobLockService distributedJobLockService;
/** 损耗闸:字段注入避免改动手工构造的调度器用例;null=未启用(单测)。 */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
@Scheduled(fixedDelayString = "${aiimage.collect-data.brand-recheck.scan-interval-ms:300000}",
initialDelayString = "${aiimage.collect-data.brand-recheck.scan-initial-delay-ms:120000}")
public void scan() {
@@ -124,6 +128,13 @@ public class CollectDataBrandRecheckScheduler {
}
return new GroupRound(0, 0);
}
if (taskGuard != null && taskGuard.isPaused(task.getUserId())) {
// 暂停不消耗尝试次数:用户排查完点继续后,这批条目应按原退避节奏重新到期
recheckService.postponeWithoutAttempt(entries, properties.getIntervalBaseMinutes());
log.info("[brand-recheck] 账号任务已暂停,整组顺延 taskId={} userId={} 条目={}",
taskId, task.getUserId(), entries.size());
return new GroupRound(0, 0);
}
if (!STATUS_SUCCESS.equals(task.getStatus()) && !STATUS_FAILED.equals(task.getStatus())) {
// 任务仍在跑:补测只做终态之后的兜底,顺延不计数(避免与分片重提竞态)
recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes());
@@ -60,6 +60,10 @@ public class NotificationService {
public static final String SCENE_SERVICE_DOWN = "service_down";
/** 麦象(18960 任务调度)异常:任务停滞/失败/队列积压。 */
public static final String SCENE_MAIXIANG_ANOMALY = "maixiang_anomaly";
/** 代理/LLM 消耗异常,账号任务已被自动暂停。 */
public static final String SCENE_COST_GUARD = "cost_guard";
/** 同一把代理/LLM 密钥被过多账号共用。 */
public static final String SCENE_SECRET_SHARED = "secret_shared";
public static final String SCENE_SYSTEM = "system";
/**
@@ -78,10 +82,12 @@ public class NotificationService {
private static final Map<String, String> SCENE_TO_CATEGORY = Map.of(
SCENE_SERVICE_DOWN, CATEGORY_SYSTEM_ERROR,
SCENE_MAIXIANG_ANOMALY, CATEGORY_SYSTEM_ERROR,
SCENE_COST_GUARD, CATEGORY_SYSTEM_ERROR,
SCENE_TASK_FAILED, CATEGORY_TASK_ERROR,
SCENE_TASK_PARTIAL, CATEGORY_TASK_ERROR,
SCENE_SECRET_BALANCE, CATEGORY_CONFIG_ERROR,
SCENE_SECRET_INVALID, CATEGORY_CONFIG_ERROR,
SCENE_SECRET_SHARED, CATEGORY_CONFIG_ERROR,
SCENE_SYSTEM, CATEGORY_SYSTEM_NOTICE);
private static final long MAX_PAGE_SIZE = 100L;
@@ -58,6 +58,10 @@ public class PriceTrackLoopRunService {
private final PriceTrackLoopRunMapper loopRunMapper;
private final FileTaskMapper fileTaskMapper;
private final ObjectMapper objectMapper;
/** 损耗闸:字段注入避免改动手工 new 的 7 处测试构造器;null=未启用(单测)。 */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
@Transactional
@@ -104,6 +108,11 @@ public class PriceTrackLoopRunService {
@Transactional
public PriceTrackLoopRunDispatchVo dispatchNext(Long loopRunId, Long userId) {
if (taskGuard != null && taskGuard.isPaused(userId)) {
// 不把循环标成 STOPPED:暂停只停派发,恢复后同一条循环还能接着跑
throw new BusinessException(com.nanri.aiimage.modules.usersecret.service
.UserTaskGuardService.PAUSED_MESSAGE);
}
PriceTrackLoopRunEntity entity = getOwnedLoopRun(loopRunId, userId);
reconcileWithTerminalChild(entity);
if (isTerminal(entity.getStatus())) {
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmKeyGate;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
import com.nanri.aiimage.config.HttpClientPool;
@@ -50,6 +51,10 @@ public class SimilarAsinLlmClient {
private final ObjectMapper objectMapper;
private final ExternalCallMetricsRecorder externalCallMetrics;
private final UserSecretUsageService userSecretUsageService;
/** 损耗闸:字段注入避免改动手工构造的 LLM 用例;null=未启用(单测)。 */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard;
private final UnreachableCooldown unreachableCooldown;
private volatile RestClient sharedRestClient;
@@ -97,11 +102,23 @@ public class SimilarAsinLlmClient {
Exception lastFailure = null;
int connectFailures = 0;
for (int attempt = 1; attempt <= attempts; attempt++) {
try {
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire(resolvedKey)) {
String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
unreachableCooldown.recordConnected(host);
return content;
} catch (Exception ex) {
if (ex instanceof com.nanri.aiimage.common.exception.TaskPausedException paused) {
// 暂停是账号级结论,重试不会变好;已完成的行走正常落库
throw paused;
}
if (ex instanceof LlmKeyGate.QueueTimeoutException) {
// 本地 key 容量已满:重试只会继续排队,直接交给行级失败
throw (LlmKeyGate.QueueTimeoutException) ex;
}
if (LlmKeyGate.isRateLimited(ex)) {
// 429 时把同 key 的下一次许可整体后移,避免所有调用一起再撞
LlmKeyGate.penalizeRateLimited(resolvedKey, 2_000L);
}
lastFailure = ex;
if (UnreachableCooldown.isConnectFailure(ex)) {
connectFailures++;
@@ -344,6 +361,10 @@ public class SimilarAsinLlmClient {
if (context == null) {
return;
}
if (taskGuard != null && taskGuard.isPaused(context.userId())) {
throw new com.nanri.aiimage.common.exception.TaskPausedException(
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE);
}
userSecretUsageService.record(context.userId(), UserSecretModule.SIMILAR_ASIN.key(), 1);
}
@@ -0,0 +1,44 @@
package com.nanri.aiimage.modules.usersecret.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.modules.usersecret.model.vo.CostGuardStatusVo;
import com.nanri.aiimage.modules.usersecret.service.CostGuardService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* 桌面端损耗监控:身份一律取自 JWT,不接受前端传 uid。
* 前端每 30 秒轮询 status;暂停时弹窗,用户排查后点「继续任务」调 resume。
*/
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/user/cost-guard")
@Tag(name = "损耗监控(桌面端)", description = "代理/LLM 异常消耗的暂停状态与手动继续。")
public class UserCostGuardController {
private final CostGuardService costGuardService;
private final AdminAuthSupport adminAuthSupport;
@GetMapping("/status")
@Operation(summary = "读取当前账号的损耗监控状态")
public ApiResponse<CostGuardStatusVo> status(HttpServletRequest request) {
return ApiResponse.success(costGuardService.status(currentUserId(request)));
}
@PostMapping("/resume")
@Operation(summary = "排查后继续任务", description = "只清暂停闸与损耗基线,不删除已完成任务和结果行。")
public ApiResponse<CostGuardStatusVo> resume(HttpServletRequest request) {
return ApiResponse.success("已继续任务", costGuardService.resume(currentUserId(request)));
}
private Long currentUserId(HttpServletRequest request) {
return adminAuthSupport.requireUser(request).getId();
}
}
@@ -1,11 +1,14 @@
package com.nanri.aiimage.modules.usersecret.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate;
import com.nanri.aiimage.modules.usersecret.model.entity.UserSecretUsageEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.time.LocalDate;
import java.util.List;
@Mapper
public interface UserSecretUsageMapper extends BaseMapper<UserSecretUsageEntity> {
@@ -33,6 +36,35 @@ public interface UserSecretUsageMapper extends BaseMapper<UserSecretUsageEntity>
@Param("promptTokens") long promptTokens,
@Param("completionTokens") long completionTokens);
/**
* 当日用量按用户 × 模块聚合(损耗扫描每分钟读一次)。
* 表是「用户 × 模块 × 天」的日聚合,行数 = 用户数 × 3,全量 GROUP BY 成本可忽略。
*/
@Select("""
SELECT user_id AS userId,
module_key AS moduleKey,
COALESCE(SUM(call_count), 0) AS callCount,
COALESCE(SUM(prompt_tokens + completion_tokens), 0) AS tokens
FROM biz_user_secret_usage_daily
WHERE business_date = #{businessDate}
GROUP BY user_id, module_key
""")
List<UserSecretUsageAggregate> sumByUserAndModule(@Param("businessDate") LocalDate businessDate);
/** 单个用户的当日读数:客户端状态接口每 30 秒轮询一次,不能为此全表 GROUP BY。 */
@Select("""
SELECT user_id AS userId,
module_key AS moduleKey,
COALESCE(SUM(call_count), 0) AS callCount,
COALESCE(SUM(prompt_tokens + completion_tokens), 0) AS tokens
FROM biz_user_secret_usage_daily
WHERE business_date = #{businessDate}
AND user_id = #{userId}
GROUP BY user_id, module_key
""")
List<UserSecretUsageAggregate> sumForUser(@Param("userId") Long userId,
@Param("businessDate") LocalDate businessDate);
/** 分批删除保留期外的用量日统计行(该表此前只增不删)。 */
int deleteBefore(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize);
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.usersecret.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
@Mapper
public interface UserTaskGuardMapper extends BaseMapper<UserTaskGuardEntity> {
/** 幂等暂停:无记录则建,有记录则刷新原因与时刻。 */
@Insert("""
INSERT INTO biz_user_task_guard (user_id, paused, reason, paused_at, updated_at)
VALUES (#{userId}, 1, #{reason}, NOW(), NOW())
ON DUPLICATE KEY UPDATE paused = 1, reason = #{reason},
paused_at = NOW(), updated_at = NOW()
""")
int pause(@Param("userId") Long userId, @Param("reason") String reason);
@Update("""
UPDATE biz_user_task_guard
SET paused = 0, reason = '', paused_at = NULL, updated_at = NOW()
WHERE user_id = #{userId}
""")
int resume(@Param("userId") Long userId);
}
@@ -0,0 +1,13 @@
package com.nanri.aiimage.modules.usersecret.model.dto;
import lombok.Data;
/** 按用户 × 模块聚合的当日用量读数(CostGuardScheduler 的扫描输入)。 */
@Data
public class UserSecretUsageAggregate {
private Long userId;
private String moduleKey;
private Long callCount;
private Long tokens;
}
@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.usersecret.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
/** 账号级任务暂停闸:损耗异常时停新派发,排查后由用户手动恢复。 */
@Data
@TableName("biz_user_task_guard")
public class UserTaskGuardEntity {
@TableId(value = "user_id", type = IdType.INPUT)
private Long userId;
private Boolean paused;
private String reason;
private LocalDateTime pausedAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,49 @@
package com.nanri.aiimage.modules.usersecret.model.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.time.LocalDateTime;
import java.util.List;
/** 客户端「损耗监控」状态:暂停闸 + 当日用量 + 本轮增量 + 共享密钥提示。 */
@Data
public class CostGuardStatusVo {
private boolean paused;
private String reason;
private LocalDateTime pausedAt;
private TodayUsage today;
private WindowUsage window;
private Thresholds thresholds;
private List<SharedSecretWarningVo> sharedWarnings;
@Data
@AllArgsConstructor
public static class TodayUsage {
private long proxy;
private long llmCalls;
private long llmTokens;
}
@Data
@AllArgsConstructor
public static class WindowUsage {
private int seconds;
private long proxy;
private long llmCalls;
private long llmTokens;
}
@Data
@AllArgsConstructor
public static class Thresholds {
private int proxyWindow;
private int llmCallsWindow;
private long llmTokensWindow;
private int proxyDaily;
private int llmCallsDaily;
private long llmTokensDaily;
private int sharedAccounts;
}
}
@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.usersecret.model.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
/** 一把代理/LLM 密钥被多个账号共用的提示(只报账号数,不暴露其他账号身份)。 */
@Data
@AllArgsConstructor
public class SharedSecretWarningVo {
/** proxy / llm */
private String type;
private String moduleLabel;
private int accounts;
private int threshold;
private String message;
}
@@ -0,0 +1,225 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.CostGuardProperties;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.notification.service.NotificationService;
import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper;
import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate;
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDate;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 代理 / LLM 损耗扫描:每分钟读一次当日用量聚合表,用「本轮增量」判失控、用「日累计跨阈值」
* 判慢烧;命中即置账号暂停闸并推用户通知。
*
* <p>为什么用日聚合表的增量而不是另建明细表:代理计次本来就是 16890/pt_worker 按批上报的,
* LLM 计次在发请求前已落库;再建一条实时流水只会多一份可能对不上的账。日表行数 = 用户数×3,
* 每分钟全量 GROUP BY 的成本可以忽略。
*
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class CostGuardScheduler {
static final String LOCK_NAME = "cost-guard-scan";
private final CostGuardProperties properties;
private final UserSecretUsageMapper usageMapper;
private final UserTaskGuardService guardService;
private final SharedSecretMonitorService sharedSecretMonitorService;
private final NotificationDispatchService notificationDispatchService;
private final DistributedJobLockService distributedJobLockService;
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
private volatile long lastSharedRefreshAt = 0L;
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
public void scan() {
if (!properties.isEnabled()) {
return;
}
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
if (lock == null) {
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
return;
}
try (lock) {
runOnce();
} catch (Exception ex) {
log.warn("[cost-guard] 损耗扫描异常终止 err={}", ex.getMessage(), ex);
}
}
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
void runOnce() {
Map<Long, UserTotals> current = loadTodayTotals();
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
int paused = 0;
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
Long userId = entry.getKey();
UserTotals now = entry.getValue();
UserTotals before = previousTotals.get(userId);
if (before == null) {
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
deltas.put(userId, WindowDelta.ZERO);
continue;
}
WindowDelta delta = new WindowDelta(
Math.max(0, now.proxy - before.proxy),
Math.max(0, now.llmCalls - before.llmCalls),
Math.max(0, now.llmTokens - before.llmTokens));
deltas.put(userId, delta);
if (guardService.isPaused(userId)) {
continue;
}
String reason = breachReason(now, before, delta);
if (reason == null) {
continue;
}
guardService.pause(userId, reason);
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
null);
paused++;
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
}
previousTotals.clear();
previousTotals.putAll(current);
lastDeltas.clear();
lastDeltas.putAll(deltas);
maybeRefreshShared();
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
}
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
public void resetBaseline(Long userId) {
if (userId == null) {
return;
}
previousTotals.remove(userId);
lastDeltas.remove(userId);
}
public WindowDelta lastDelta(Long userId) {
WindowDelta delta = userId == null ? null : lastDeltas.get(userId);
return delta == null ? WindowDelta.ZERO : delta;
}
public Map<Long, UserTotals> loadTodayTotals() {
List<UserSecretUsageAggregate> rows = usageMapper.sumByUserAndModule(LocalDate.now());
Map<Long, UserTotals> totals = new LinkedHashMap<>();
for (UserSecretUsageAggregate row : rows) {
if (row.getUserId() == null || row.getUserId() <= 0) {
continue;
}
accumulate(totals.computeIfAbsent(row.getUserId(), key -> new UserTotals()), row);
}
return totals;
}
private void accumulate(UserTotals item, UserSecretUsageAggregate row) {
long calls = row.getCallCount() == null ? 0L : row.getCallCount();
long tokens = row.getTokens() == null ? 0L : row.getTokens();
if (UserSecretModule.PROXY.key().equals(row.getModuleKey())) {
item.proxy += calls;
} else {
item.llmCalls += calls;
item.llmTokens += tokens;
}
}
public UserTotals todayTotals(Long userId) {
if (userId == null || userId <= 0) {
return new UserTotals();
}
try {
UserTotals totals = new UserTotals();
for (UserSecretUsageAggregate row : usageMapper.sumForUser(userId, LocalDate.now())) {
accumulate(totals, row);
}
return totals;
} catch (Exception ex) {
// 状态接口是客户端每 30 秒一次的轮询:读不到就返回零值,不让一次库抖动打断提示
log.warn("[cost-guard] 读取用户当日用量失败 userId={} err={}", userId, ex.getMessage());
return new UserTotals();
}
}
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
int seconds = Math.max(60, properties.getWindowSeconds());
if (properties.getProxyWindowThreshold() > 0
&& delta.proxy >= properties.getProxyWindowThreshold()) {
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
+ properties.getProxyWindowThreshold() + " 次";
}
if (properties.getLlmWindowCallsThreshold() > 0
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
+ properties.getLlmWindowCallsThreshold() + " 次";
}
if (properties.getLlmWindowTokensThreshold() > 0
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
+ properties.getLlmWindowTokensThreshold();
}
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
+ properties.getDailyProxyThreshold() + " 次";
}
if (crossed(before.llmCalls, now.llmCalls, properties.getDailyLlmCallsThreshold())) {
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
+ properties.getDailyLlmCallsThreshold() + " 次";
}
if (crossed(before.llmTokens, now.llmTokens, properties.getDailyLlmTokensThreshold())) {
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
+ properties.getDailyLlmTokensThreshold();
}
return null;
}
private boolean crossed(long before, long now, long threshold) {
return threshold > 0 && before < threshold && now >= threshold;
}
private void maybeRefreshShared() {
long now = System.currentTimeMillis();
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
return;
}
lastSharedRefreshAt = now;
try {
sharedSecretMonitorService.refresh();
} catch (Exception ex) {
// 共享密钥扫描失败不影响损耗暂停主流程,下一轮再试
log.warn("[cost-guard] 共享密钥扫描失败 err={}", ex.getMessage(), ex);
}
}
public static final class UserTotals {
public long proxy;
public long llmCalls;
public long llmTokens;
}
public record WindowDelta(long proxy, long llmCalls, long llmTokens) {
public static final WindowDelta ZERO = new WindowDelta(0, 0, 0);
}
}
@@ -0,0 +1,53 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.config.CostGuardProperties;
import com.nanri.aiimage.modules.usersecret.model.vo.CostGuardStatusVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
/** 客户端损耗监控的状态读取与手动继续;不改任何任务结果行。 */
@Service
@RequiredArgsConstructor
public class CostGuardService {
private final CostGuardProperties properties;
private final UserTaskGuardService guardService;
private final CostGuardScheduler costGuardScheduler;
private final SharedSecretMonitorService sharedSecretMonitorService;
public CostGuardStatusVo status(Long userId) {
UserTaskGuardService.GuardState state = guardService.state(userId);
CostGuardScheduler.UserTotals today = costGuardScheduler.todayTotals(userId);
CostGuardScheduler.WindowDelta delta = costGuardScheduler.lastDelta(userId);
int windowSeconds = Math.max(60, properties.getWindowSeconds());
CostGuardStatusVo vo = new CostGuardStatusVo();
vo.setPaused(state.paused());
vo.setReason(state.reason());
vo.setPausedAt(state.pausedAt());
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
vo.setThresholds(new CostGuardStatusVo.Thresholds(
properties.getProxyWindowThreshold(),
properties.getLlmWindowCallsThreshold(),
properties.getLlmWindowTokensThreshold(),
properties.getDailyProxyThreshold(),
properties.getDailyLlmCallsThreshold(),
properties.getDailyLlmTokensThreshold(),
properties.getSharedAccountsThreshold()));
List<com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo> shared =
sharedSecretMonitorService.warningsFor(userId);
vo.setSharedWarnings(shared == null ? List.of() : shared);
return vo;
}
/** 用户排查后手动继续:清暂停位并把损耗基线对齐到当前,已完成的任务与结果原样保留。 */
public CostGuardStatusVo resume(Long userId) {
guardService.resume(userId);
costGuardScheduler.resetBaseline(userId);
return status(userId);
}
}
@@ -0,0 +1,150 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.nanri.aiimage.config.CostGuardProperties;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.notification.service.NotificationService;
import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper;
import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity;
import com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo;
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* 共享密钥提示:同一把代理/LLM 密钥被过多账号共用时,给每个使用者一条明确提示。
*
* <p>只统计、不暂停:共用同一把密钥本身不是故障(账单按 key 计费,平台也允许这样配),
* 但它会把一次异常消耗放大到所有账号,也会让「谁烧的」无法归因。提示里只报账号数,
* 不暴露其他账号身份。
*
* <p>分组键直接用密文:{@code ShopCredentialCryptoService} 是 AES/ECB 无 IV,
* 同明文必同密文(与 {@link UserApiSecretService#checkAllForScheduler} 同一口径),
* 因此本类不需要解密、也不会把明文密钥带进内存快照。
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class SharedSecretMonitorService {
private final UserApiSecretMapper userApiSecretMapper;
private final CostGuardProperties properties;
private final NotificationDispatchService notificationDispatchService;
private volatile Map<Long, List<SharedSecretWarningVo>> warningsByUser = Map.of();
private volatile long snapshotAtMillis = 0L;
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
public synchronized void refresh() {
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
warningsByUser = Map.of();
snapshotAtMillis = System.currentTimeMillis();
return;
}
// 用字符串列名而不是 lambda:本类在纯单测里不经过 MyBatis 的实体元数据初始化
List<UserApiSecretEntity> rows = userApiSecretMapper.selectList(
new QueryWrapper<UserApiSecretEntity>()
.select("user_id", "module_key", "secret_value"));
// groupKey = module|密文;值是该组的使用账号集合(去重)
Map<String, SecretGroup> groups = new LinkedHashMap<>();
for (UserApiSecretEntity row : rows) {
if (row.getUserId() == null || row.getUserId() <= 0) {
continue;
}
Optional<UserSecretModule> module = UserSecretModule.of(row.getModuleKey());
if (module.isEmpty() || row.getSecretValue() == null || row.getSecretValue().isBlank()) {
continue;
}
String groupKey = row.getModuleKey() + '|' + row.getSecretValue();
groups.computeIfAbsent(groupKey, key -> new SecretGroup(module.get()))
.userIds.add(row.getUserId());
}
Map<Long, List<SharedSecretWarningVo>> next = new LinkedHashMap<>();
int hitGroups = 0;
for (Map.Entry<String, SecretGroup> entry : groups.entrySet()) {
SecretGroup group = entry.getValue();
int accounts = group.userIds.size();
if (accounts < properties.getSharedAccountsThreshold()) {
continue;
}
hitGroups++;
String type = group.module == UserSecretModule.PROXY ? "proxy" : "llm";
int threshold = properties.getSharedAccountsThreshold();
String message = buildMessage(group.module, accounts, threshold);
SharedSecretWarningVo warning = new SharedSecretWarningVo(
type, group.module.label(), accounts, threshold, message);
for (Long userId : group.userIds) {
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
pushDailyWarning(userId, group.module, entry.getKey(), warning);
}
}
warningsByUser = next;
snapshotAtMillis = System.currentTimeMillis();
log.info("[cost-guard] 共享密钥扫描完成 配置行={} 密钥组={} 超阈值组={} 受影响账号={}",
rows.size(), groups.size(), hitGroups, next.size());
}
/** 当前用户共享密钥提示;快照未建立时返回空,不阻塞客户端轮询。 */
public List<SharedSecretWarningVo> warningsFor(Long userId) {
if (userId == null || userId <= 0) {
return List.of();
}
return warningsByUser.getOrDefault(userId, List.of());
}
public long snapshotAtMillis() {
return snapshotAtMillis;
}
private void pushDailyWarning(Long userId, UserSecretModule module, String groupKey,
SharedSecretWarningVo warning) {
String day = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
String dedupe = "secret_shared:" + module.key() + ':' + sha256Hex(groupKey)
+ ':' + userId + ':' + day;
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_SECRET_SHARED,
NotificationService.LEVEL_WARNING,
module.label() + "被 " + warning.getAccounts() + " 个账号共用",
warning.getMessage(), dedupe);
}
private String buildMessage(UserSecretModule module, int accounts, int threshold) {
String what = module == UserSecretModule.PROXY ? "代理提取链接" : "LLM 密钥";
return "你的" + module.label() + "(" + what + ")当前被 " + accounts
+ " 个账号共用,已达到提示阈值 " + threshold + "。任一账号的异常消耗都会算在这把密钥的套餐上,"
+ "建议在「密钥设置」里为不同账号拆分独立密钥,便于控费和定位问题。";
}
private String sha256Hex(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8)));
} catch (Exception ex) {
return Integer.toHexString(value.hashCode());
}
}
private static final class SecretGroup {
private final UserSecretModule module;
private final Set<Long> userIds = new HashSet<>();
private SecretGroup(UserSecretModule module) {
this.module = module;
}
}
}
@@ -0,0 +1,118 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.modules.usersecret.mapper.UserTaskGuardMapper;
import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
/**
* 账号级任务暂停闸:损耗异常时由 CostGuardScheduler 置位,派发入口据此停新任务。
*
* <p>读侧带 3 秒内存缓存:品牌检测/LLM 每次请求都问一次,直接打库会把闸门本身变成热点;
* 多实例下最迟 3 秒收敛,足够挡住下一分钟的任务。写侧立即清缓存,本机无延迟。
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class UserTaskGuardService {
public static final String PAUSED_MESSAGE =
"账号任务已暂停(损耗保护),请排查代理/LLM 消耗后在客户端点击「继续任务」";
private static final long CACHE_TTL_NANOS = TimeUnit.SECONDS.toNanos(3);
private static final int REASON_MAX_LENGTH = 500;
private final UserTaskGuardMapper guardMapper;
private final ConcurrentHashMap<Long, CachedState> cache = new ConcurrentHashMap<>();
public boolean isPaused(Long userId) {
if (userId == null || userId <= 0) {
return false;
}
CachedState cached = cache.get(userId);
if (cached != null && System.nanoTime() - cached.readAtNanos() < CACHE_TTL_NANOS) {
return cached.paused();
}
GuardState state = load(userId);
cache.put(userId, new CachedState(state.paused(), state.reason(), state.pausedAt(),
System.nanoTime()));
return state.paused();
}
/** 状态查询走库:客户端弹窗要看到其他实例刚写入的暂停,不吃本机缓存。 */
public GuardState state(Long userId) {
if (userId == null || userId <= 0) {
return GuardState.ACTIVE;
}
GuardState state = load(userId);
cache.put(userId, new CachedState(state.paused(), state.reason(), state.pausedAt(),
System.nanoTime()));
return state;
}
public void pause(Long userId, String reason) {
if (userId == null || userId <= 0) {
return;
}
String safeReason = truncate(reason);
try {
guardMapper.pause(userId, safeReason);
cache.remove(userId);
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
} catch (Exception ex) {
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
userId, safeReason, ex.getMessage(), ex);
}
}
/** 用户手动继续:只清暂停位,不删任何已完成的任务/结果。 */
public boolean resume(Long userId) {
if (userId == null || userId <= 0) {
return false;
}
try {
guardMapper.resume(userId);
cache.remove(userId);
log.info("[cost-guard] 用户手动继续任务 userId={}", userId);
return true;
} catch (Exception ex) {
log.error("[cost-guard] 继续任务写入失败 userId={} err={}", userId, ex.getMessage(), ex);
return false;
}
}
private GuardState load(Long userId) {
try {
UserTaskGuardEntity row = guardMapper.selectById(userId);
if (row == null || !Boolean.TRUE.equals(row.getPaused())) {
return GuardState.ACTIVE;
}
return new GuardState(true, row.getReason() == null ? "" : row.getReason(),
row.getPausedAt());
} catch (Exception ex) {
// 读不到闸门状态时按「不暂停」走:数据库抖动不该把全体用户的合法任务一起冻住。
// 真正的失控由下一轮扫描重新置位。
log.warn("[cost-guard] 读取暂停状态失败,按未暂停处理 userId={} err={}",
userId, ex.getMessage());
return GuardState.ACTIVE;
}
}
private String truncate(String reason) {
String value = reason == null ? "" : reason.trim();
return value.length() <= REASON_MAX_LENGTH ? value : value.substring(0, REASON_MAX_LENGTH);
}
private record CachedState(boolean paused, String reason, LocalDateTime pausedAt, long readAtNanos) {
}
public record GuardState(boolean paused, String reason, LocalDateTime pausedAt) {
public static final GuardState ACTIVE = new GuardState(false, "", null);
}
}
@@ -9,6 +9,13 @@ AIIMAGE_INSTANCE_ID=local-121
AIIMAGE_SERVER_PORT=18080
AIIMAGE_INTERNAL_TOKEN=change-me-local-internal-token
# 代理/LLM 损耗保护:异常放大时暂停该账号新任务;共享密钥超过阈值只提示不暂停。
AIIMAGE_COST_GUARD_ENABLED=true
AIIMAGE_COST_GUARD_PROXY_WINDOW_THRESHOLD=600
AIIMAGE_COST_GUARD_LLM_WINDOW_CALLS_THRESHOLD=180
AIIMAGE_COST_GUARD_LLM_WINDOW_TOKENS_THRESHOLD=600000
AIIMAGE_COST_GUARD_SHARED_ACCOUNTS_THRESHOLD=3
AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=change-me
AIIMAGE_DB_PASSWORD=change-me
@@ -501,6 +501,21 @@ aiimage:
archive-connect-timeout-millis: ${AIIMAGE_IMAGE_VIDEO_ARCHIVE_CONNECT_TIMEOUT_MILLIS:10000}
archive-read-timeout-millis: ${AIIMAGE_IMAGE_VIDEO_ARCHIVE_READ_TIMEOUT_MILLIS:600000}
archive-max-attempts: ${AIIMAGE_IMAGE_VIDEO_ARCHIVE_MAX_ATTEMPTS:3}
# 代理/LLM 损耗保护:分钟增量与当日累计双阈值,命中即暂停该账号新任务;
# 共享密钥阈值只提示不暂停(共用本身不是故障,但会把异常消耗放大到所有账号)。
cost-guard:
enabled: ${AIIMAGE_COST_GUARD_ENABLED:true}
scan-interval-ms: ${AIIMAGE_COST_GUARD_SCAN_INTERVAL_MS:60000}
scan-initial-delay-ms: ${AIIMAGE_COST_GUARD_SCAN_INITIAL_DELAY_MS:30000}
window-seconds: ${AIIMAGE_COST_GUARD_WINDOW_SECONDS:60}
proxy-window-threshold: ${AIIMAGE_COST_GUARD_PROXY_WINDOW_THRESHOLD:600}
llm-window-calls-threshold: ${AIIMAGE_COST_GUARD_LLM_WINDOW_CALLS_THRESHOLD:180}
llm-window-tokens-threshold: ${AIIMAGE_COST_GUARD_LLM_WINDOW_TOKENS_THRESHOLD:600000}
daily-proxy-threshold: ${AIIMAGE_COST_GUARD_DAILY_PROXY_THRESHOLD:8000}
daily-llm-calls-threshold: ${AIIMAGE_COST_GUARD_DAILY_LLM_CALLS_THRESHOLD:6000}
daily-llm-tokens-threshold: ${AIIMAGE_COST_GUARD_DAILY_LLM_TOKENS_THRESHOLD:20000000}
shared-accounts-threshold: ${AIIMAGE_COST_GUARD_SHARED_ACCOUNTS_THRESHOLD:3}
shared-scan-interval-ms: ${AIIMAGE_COST_GUARD_SHARED_SCAN_INTERVAL_MS:600000}
user-secret:
check-enabled: ${AIIMAGE_USER_SECRET_CHECK_ENABLED:true}
check-cron: ${AIIMAGE_USER_SECRET_CHECK_CRON:0 30 4 * * *}
@@ -0,0 +1,30 @@
-- =============================================================
-- 迁移 V168__user_task_guard
--
-- 1. 变更目的:新增「账号级任务暂停闸」。
-- - 代理提取 / LLM 调用在短时间内异常放大(或同一把密钥被过多账号共用)时,
-- 由 CostGuardScheduler 自动置 paused=1,所有派发入口据此停新任务;
-- - 用户在客户端排查后点「继续任务」,走 resume 接口置 paused=0。
-- - 暂停只停新派发,不删除、不回滚已完成的任务与结果行。
--
-- 2. 口径:
-- - 一行一个 user_id(主键),无记录 = 未暂停;
-- - reason 存给用户看的暂停原因(含触发数值),最长 512;
-- - paused_at 只记录最后一次自动暂停时刻,恢复时清空。
--
-- 3. 影响表与数据量:新表 1 张,空表,无存量数据迁移。
--
-- 4. 【可重跑性】CREATE TABLE IF NOT EXISTS,重跑 no-op。
--
-- 5. 回滚(如需):DROP TABLE IF EXISTS `biz_user_task_guard`;
-- =============================================================
CREATE TABLE IF NOT EXISTS `biz_user_task_guard` (
`user_id` BIGINT NOT NULL COMMENT '账号 id',
`paused` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '1=暂停新任务派发',
`reason` VARCHAR(512) NOT NULL DEFAULT '' COMMENT '暂停原因(展示给用户)',
`paused_at` DATETIME NULL COMMENT '最后一次自动暂停时刻',
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`user_id`),
KEY `idx_user_task_guard_paused` (`paused`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='账号级任务暂停闸(损耗保护)';