From 9c4905c94ed43c9a155bd22bca81464e928899bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 24 Oct 2026 10:50:20 +0800 Subject: [PATCH] =?UTF-8?q?feat(cost-guard):=20=E4=BB=A3=E7=90=86/LLM=20?= =?UTF-8?q?=E6=8D=9F=E8=80=97=E7=9B=91=E6=8E=A7=E4=B8=8E=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E7=BA=A7=E4=BB=BB=E5=8A=A1=E6=9A=82=E5=81=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 每分钟扫描用量增量/日累计,异常置暂停闸并推站内通知 - 品牌补测、跟价派发、外观激活、品牌检测与两个 LLM 客户端统一接闸 - 同一代理/LLM 密钥被 3 个以上账号共用时给出用户提示 - 品牌检测出口拒绝批级熔断;LLM 同 key 共享并发/间隔/429 惩罚 - V170 新增 biz_user_task_guard --- .../common/exception/TaskPausedException.java | 12 + .../aiimage/common/retry/LlmKeyGate.java | 186 +++++++++++++++ .../aiimage/config/CostGuardProperties.java | 52 ++++ .../aiimage/config/PropertiesConfig.java | 2 +- .../client/AppearancePatentLlmClient.java | 21 +- .../service/AppearancePatentTaskService.java | 8 + .../brand/client/BrandCheckClient.java | 53 ++++- .../support/BrandCheckFailureNotice.java | 6 + .../CollectDataBrandRecheckScheduler.java | 11 + .../service/NotificationService.java | 6 + .../service/PriceTrackLoopRunService.java | 9 + .../client/SimilarAsinLlmClient.java | 23 +- .../controller/UserCostGuardController.java | 44 ++++ .../mapper/UserSecretUsageMapper.java | 32 +++ .../mapper/UserTaskGuardMapper.java | 28 +++ .../model/dto/UserSecretUsageAggregate.java | 13 + .../model/entity/UserTaskGuardEntity.java | 21 ++ .../model/vo/CostGuardStatusVo.java | 49 ++++ .../model/vo/SharedSecretWarningVo.java | 17 ++ .../service/CostGuardScheduler.java | 225 ++++++++++++++++++ .../usersecret/service/CostGuardService.java | 53 +++++ .../service/SharedSecretMonitorService.java | 150 ++++++++++++ .../service/UserTaskGuardService.java | 118 +++++++++ .../resources/application-local.example.yml | 7 + .../src/main/resources/application.yml | 15 ++ .../resources/db/V170__user_task_guard.sql | 30 +++ .../aiimage/common/retry/LlmKeyGateTest.java | 38 +++ .../brand/client/BrandCheckClientTest.java | 79 ++++++ .../service/PriceTrackLoopRunServiceTest.java | 18 ++ .../service/CostGuardSchedulerTest.java | 137 +++++++++++ .../SharedSecretMonitorServiceTest.java | 101 ++++++++ .../service/UserTaskGuardServiceTest.java | 66 +++++ 32 files changed, 1625 insertions(+), 5 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/common/exception/TaskPausedException.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmKeyGate.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/config/CostGuardProperties.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/UserCostGuardController.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserTaskGuardMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageAggregate.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserTaskGuardEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/CostGuardStatusVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/SharedSecretWarningVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardScheduler.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardService.java create mode 100644 backend-java/src/main/resources/db/V170__user_task_guard.sql create mode 100644 backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmKeyGateTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/CostGuardSchedulerTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorServiceTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardServiceTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/exception/TaskPausedException.java b/backend-java/src/main/java/com/nanri/aiimage/common/exception/TaskPausedException.java new file mode 100644 index 000000000..39988c9bd --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/exception/TaskPausedException.java @@ -0,0 +1,12 @@ +package com.nanri.aiimage.common.exception; + +/** + * 账号任务已被损耗保护暂停:调用方必须立即停止重试(这不是上游抖动,重试不会变好), + * 保留已完成数据,等用户在客户端点「继续任务」后再重跑剩余部分。 + */ +public class TaskPausedException extends RuntimeException { + + public TaskPausedException(String message) { + super(message); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmKeyGate.java b/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmKeyGate.java new file mode 100644 index 000000000..cd38014bf --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmKeyGate.java @@ -0,0 +1,186 @@ +package com.nanri.aiimage.common.retry; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.client.HttpStatusCodeException; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 同一把 LLM key 的进程内全局请求门:并发上限 + 最小间隔 + 429 共享冷却。 + * + *

多个账号配同一把 key 时,provider 的 RPM/TPM 是共享的;如果每个调用各自 + * 重试,429 会互相放大。这里把并发和节奏收到 key 级别,让排队替代重试: + * 正常时几乎不等待,打满时后来者按顺序等,避免全员 429。 + * + *

配置(环境变量,默认保守;按 provider 实际额度调): + *

+ * + *

只存 key 的 SHA-256 指纹,不把原文写进 map/日志。 + */ +@Slf4j +public final class LlmKeyGate { + + private static final int MAX_CONCURRENCY = positiveInt("AIIMAGE_LLM_KEY_MAX_CONCURRENCY", 8); + private static final long MIN_INTERVAL_NANOS = + TimeUnit.MILLISECONDS.toNanos(positiveLong("AIIMAGE_LLM_KEY_MIN_INTERVAL_MS", 50L)); + private static final long ACQUIRE_TIMEOUT_NANOS = + TimeUnit.MILLISECONDS.toNanos(positiveLong("AIIMAGE_LLM_KEY_ACQUIRE_TIMEOUT_MS", 120_000L)); + + private static final ConcurrentHashMap GATES = new ConcurrentHashMap<>(); + + private LlmKeyGate() { + } + + /** 同 key 的排队许可;close 必须调用(try-with-resources 自动归还)。 */ + public static final class Permit implements AutoCloseable { + private final Gate gate; + + private Permit(Gate gate) { + this.gate = gate; + } + + @Override + public void close() { + gate.release(); + } + } + + /** 排队超时:这是本地容量结论,重试只会继续排队,调用方应停止阶梯。 */ + public static final class QueueTimeoutException extends IllegalStateException { + public QueueTimeoutException(String message) { + super(message); + } + } + + public static Permit acquire(String apiKey) { + Gate gate = GATES.computeIfAbsent(fingerprint(apiKey), ignored -> new Gate()); + return gate.acquire(); + } + + /** 429 时把同 key 的下一次许可整体后移,避免所有线程同时再撞一次。 */ + public static void penalizeRateLimited(String apiKey, long millis) { + GATES.computeIfAbsent(fingerprint(apiKey), ignored -> new Gate()) + .penalize(TimeUnit.MILLISECONDS.toNanos(Math.max(0L, millis))); + } + + /** 从异常链里识别 HTTP 429;其他异常(超时/5xx)不改变 key 级节奏。 */ + public static boolean isRateLimited(Throwable error) { + Throwable current = error; + while (current != null) { + if (current instanceof HttpStatusCodeException status + && status.getStatusCode().value() == 429) { + return true; + } + current = current.getCause(); + } + return false; + } + + public static void resetForTests() { + GATES.clear(); + } + + private static String fingerprint(String apiKey) { + String key = apiKey == null ? "" : apiKey.trim(); + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(key.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte b : digest) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException ex) { + // SHA-256 是 JDK 必备算法;真缺失时退化为长度+哈希码,仍不落原文 + return "fallback:" + key.length() + ":" + key.hashCode(); + } + } + + private static int positiveInt(String name, int fallback) { + try { + int value = Integer.parseInt(System.getenv().getOrDefault(name, "").trim()); + return value > 0 ? value : fallback; + } catch (RuntimeException ex) { + return fallback; + } + } + + private static long positiveLong(String name, long fallback) { + try { + long value = Long.parseLong(System.getenv().getOrDefault(name, "").trim()); + return value > 0 ? value : fallback; + } catch (RuntimeException ex) { + return fallback; + } + } + + private static final class Gate { + private final Semaphore permits = new Semaphore(MAX_CONCURRENCY); + private final Object lock = new Object(); + private final AtomicLong nextAtNanos = new AtomicLong(); + + Permit acquire() { + try { + if (!permits.tryAcquire(ACQUIRE_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) { + throw new QueueTimeoutException( + "LLM key 排队超时(同 key 并发已满,等待上限 " + + TimeUnit.NANOSECONDS.toMillis(ACQUIRE_TIMEOUT_NANOS) + "ms)"); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("LLM key 排队被中断", ex); + } + try { + waitForInterval(); + return new Permit(this); + } catch (RuntimeException ex) { + permits.release(); + throw ex; + } + } + + void release() { + permits.release(); + } + + void penalize(long nanos) { + if (nanos <= 0) { + return; + } + long now = System.nanoTime(); + nextAtNanos.accumulateAndGet(now + nanos, Math::max); + } + + private void waitForInterval() { + long interval = MIN_INTERVAL_NANOS; + while (interval > 0) { + long now = System.nanoTime(); + long next = nextAtNanos.get(); + long wait = next - now; + if (wait <= 0) { + long assigned = now + interval; + if (nextAtNanos.compareAndSet(next, assigned)) { + return; + } + continue; + } + try { + TimeUnit.NANOSECONDS.sleep(Math.min(wait, TimeUnit.MILLISECONDS.toNanos(50))); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("LLM key 间隔等待被中断", ex); + } + } + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/CostGuardProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/CostGuardProperties.java new file mode 100644 index 000000000..f9ba764ae --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/CostGuardProperties.java @@ -0,0 +1,52 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * 代理 / LLM 损耗保护:按分钟增量与当日累计两条线判定异常,命中即暂停该账号的新任务。 + * + *

阈值默认值按「正常批次不该碰、失控循环一定碰」取:品牌补测单批最多百来个品牌, + * 一分钟内正常提取几十次;失控时一个账号一分钟能烧到几百上千次。上线后按后台用量 + * 曲线调整,不把阈值写死在代码里。 + */ +@Data +@ConfigurationProperties(prefix = "aiimage.cost-guard") +public class CostGuardProperties { + + /** 总开关:关掉只停自动暂停与提示,记账与状态接口仍可用。 */ + private boolean enabled = true; + + /** 扫描间隔(毫秒);用量是按分钟增量判定的,默认 60 秒。 */ + private long scanIntervalMs = 60_000L; + + /** 首次扫描延迟(毫秒):避开应用启动时的迁移与预热。 */ + private long scanInitialDelayMs = 30_000L; + + /** 展示给用户的窗口长度(秒);扫描间隔默认 60 秒,含义是「最近约一分钟增量」。 */ + private int windowSeconds = 60; + + /** 窗口内代理提取次数阈值;超过即暂停。 */ + private int proxyWindowThreshold = 600; + + /** 窗口内 LLM 调用次数阈值。 */ + private int llmWindowCallsThreshold = 180; + + /** 窗口内 LLM token 阈值(prompt + completion)。 */ + private long llmWindowTokensThreshold = 600_000L; + + /** 当日代理提取次数上限;0=关闭。 */ + private int dailyProxyThreshold = 8_000; + + /** 当日 LLM 调用次数上限;0=关闭。 */ + private int dailyLlmCallsThreshold = 6_000; + + /** 当日 LLM token 上限;0=关闭。 */ + private long dailyLlmTokensThreshold = 20_000_000L; + + /** 同一把代理/LLM 密钥被多少个账号共用时提示;0=关闭。 */ + private int sharedAccountsThreshold = 3; + + /** 共享密钥扫描间隔(毫秒);首轮随第一次损耗扫描一起跑。 */ + private long sharedScanIntervalMs = 600_000L; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java index c44e78c8a..3c4067b8a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java @@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Configuration; @Configuration -@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class}) +@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class, CostGuardProperties.class}) public class PropertiesConfig { } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java index 841713e03..189a825f8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.LlmKeyGate; import com.nanri.aiimage.common.retry.LlmRetryBackoff; import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; @@ -156,6 +157,10 @@ public class AppearancePatentLlmClient { private final ExternalCallMetricsRecorder externalCallMetrics; private final BrandCheckClient brandCheckClient; private final UserSecretUsageService userSecretUsageService; + + /** 损耗闸:字段注入避免改动手工构造的 LLM 用例;null=未启用(单测)。 */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard; private final UnreachableCooldown unreachableCooldown; private volatile RestClient sharedRestClient; @@ -656,11 +661,21 @@ public class AppearancePatentLlmClient { Exception lastFailure = null; AtomicInteger connectFailures = new AtomicInteger(); for (int attempt = 1; attempt <= attempts; attempt++) { - try { + try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire(apiKey)) { String content = invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt); unreachableCooldown.recordConnected(host); return content; } catch (Exception ex) { + if (ex instanceof com.nanri.aiimage.common.exception.TaskPausedException paused) { + // 暂停是账号级结论,重试不会变好;已完成的行走正常落库 + throw paused; + } + if (ex instanceof LlmKeyGate.QueueTimeoutException) { + throw (LlmKeyGate.QueueTimeoutException) ex; + } + if (LlmKeyGate.isRateLimited(ex)) { + LlmKeyGate.penalizeRateLimited(apiKey, 2_000L); + } lastFailure = ex; if (UnreachableCooldown.isConnectFailure(ex)) { boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis); @@ -1351,6 +1366,10 @@ public class AppearancePatentLlmClient { if (context == null) { return; } + if (taskGuard != null && taskGuard.isPaused(context.userId())) { + throw new com.nanri.aiimage.common.exception.TaskPausedException( + com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE); + } userSecretUsageService.record(context.userId(), UserSecretModule.APPEARANCE_PATENT.key(), 1); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 8bb9deb95..c6b914b81 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -109,6 +109,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline rejectedSubmission = new ThreadLocal<>(); private volatile RestClient sharedRestClient; @@ -260,6 +278,16 @@ public class BrandCheckClient { // 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到 // 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。 Map outcomes = new LinkedHashMap<>(); + if (taskGuard != null && taskGuard.isPaused(userId)) { + // 损耗闸已落下:整批一个请求都不发,结论按查询失败留待补测;重试不会让暂停变好 + log.warn("[brand-check] 账号任务已暂停,整批不打上游 userId={} 品牌数={}", + userId, brands.size()); + for (String brand : brands) { + outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand), + com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE)); + } + return outcomes; + } String proxyUrl = proxyUrlFor(userId); if (proxyUrl.isBlank()) { // 缺代理整批一个请求都不发:这是配置问题,十条重试阶梯在这里只会白烧, @@ -293,6 +321,8 @@ public class BrandCheckClient { // 连接层证据跨品牌累计:批内任一次成功响应清零(见 checkOneBrand)。 // 一次批量一份,不跨批复用——另一批的 upstreamRejecting 也是这个理由。 AtomicInteger connectFailures = new AtomicInteger(); + // 出口被上游拒绝的证据跨品牌累计:达到阈值后不再给后续品牌发首轮探针。 + AtomicInteger egressRejections = new AtomicInteger(); for (String brand : brands) { AtomicBoolean started = new AtomicBoolean(false); AtomicBoolean rejected = new AtomicBoolean(false); @@ -303,7 +333,7 @@ public class BrandCheckClient { () -> { started.set(true); return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, - proxyUrl, userId, connectFailures); + proxyUrl, userId, connectFailures, egressRejections); }, checkExecutor); } finally { @@ -333,7 +363,8 @@ public class BrandCheckClient { private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos, AtomicBoolean upstreamRejecting, String proxyUrl, Long userId, - AtomicInteger connectFailures) { + AtomicInteger connectFailures, + AtomicInteger egressRejections) { int attempts = Math.max(1, properties.getRetryTimes()); BrandCheckResponse response = null; Exception lastFailure = null; @@ -352,6 +383,14 @@ public class BrandCheckClient { lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage()); break; } + // 出口池已被上游连续拒绝:后面的品牌连首轮探针都不发。与 upstreamRejecting + // 不同,这里允许在 attempt=1 就短路——前 2 个品牌已经各烧过出口 IP,再让 + // 每个品牌各打一次只是把同一笔失败摊成品牌数×2 的提取费。 + if (egressRejections.get() >= EGRESS_REJECTED_EVIDENCE) { + log.warn("[brand-check] 代理出口已被连续拒绝 {} 次,本品牌不再换 IP brand={} attempt={}/{}", + egressRejections.get(), brand, attempt, attempts); + return queryFailedOutcome(brand, REASON_EGRESS_REJECTED); + } // 同批已有品牌走完整个阶梯、每一轮都被判查询失败 ⇒ 上游在确定性拒绝,本品牌别再烧 // 等待与整批预算。2026-10-19 现网:品牌库代理套餐过期 → 16890 全部直连 WIPO → // 整片 Forbidden,单品牌 10 轮含递增等待约 45~65 秒全是白烧,而它烧掉的正是后面 @@ -411,6 +450,16 @@ public class BrandCheckClient { if (response.getReason() != null && !response.getReason().isBlank()) { reason = response.getReason().trim(); } + if (BrandCheckFailureNotice.isEgressRejected(reason)) { + // 这个品牌已经换过出口仍被拒:停止它自己的阶梯,并累计批级证据。 + // 16890 侧也有出口熔断,两层都只省“继续换 IP”的钱,不改“这行没查成”的结论。 + ladderCompleted = false; + upstreamRejecting.set(true); + int seen = egressRejections.incrementAndGet(); + log.warn("[brand-check] 代理出口被上游拒绝(本批第 {} 次),停止换 IP brand={} attempt={}/{} reason={}", + seen, brand, attempt, attempts, reason); + break; + } if (response != null && !response.isRetryable()) { // 上游当场判定"重试也不会变好"(供应商拒付、套餐过期、这类结论是确定的), // 阶梯到此为止并把同批一起短路——旧版这里要跑满 10 轮、约 45~65 秒才知道 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/support/BrandCheckFailureNotice.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/support/BrandCheckFailureNotice.java index b027fa2fa..f475479db 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/support/BrandCheckFailureNotice.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/support/BrandCheckFailureNotice.java @@ -70,6 +70,12 @@ public final class BrandCheckFailureNotice { return Kind.UPSTREAM; } + /** 原始原因是否属于「代理出口 IP 被上游拒绝」。批量熔断共用这一份判据。 */ + public static boolean isEgressRejected(String rawReason) { + String reason = rawReason == null ? "" : rawReason.trim(); + return !reason.isEmpty() && EGRESS_REJECTED.matcher(reason).find(); + } + /** 用户可读的那一句:先说发生了什么,再说该做什么。 */ public static String userMessage(Kind kind) { if (kind == null) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckScheduler.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckScheduler.java index f444f1d76..08511f1eb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckScheduler.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckScheduler.java @@ -59,6 +59,10 @@ public class CollectDataBrandRecheckScheduler { private final TaskFileJobService taskFileJobService; private final DistributedJobLockService distributedJobLockService; + /** 损耗闸:字段注入避免改动手工构造的调度器用例;null=未启用(单测)。 */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard; + @Scheduled(fixedDelayString = "${aiimage.collect-data.brand-recheck.scan-interval-ms:300000}", initialDelayString = "${aiimage.collect-data.brand-recheck.scan-initial-delay-ms:120000}") public void scan() { @@ -124,6 +128,13 @@ public class CollectDataBrandRecheckScheduler { } return new GroupRound(0, 0); } + if (taskGuard != null && taskGuard.isPaused(task.getUserId())) { + // 暂停不消耗尝试次数:用户排查完点继续后,这批条目应按原退避节奏重新到期 + recheckService.postponeWithoutAttempt(entries, properties.getIntervalBaseMinutes()); + log.info("[brand-recheck] 账号任务已暂停,整组顺延 taskId={} userId={} 条目={}", + taskId, task.getUserId(), entries.size()); + return new GroupRound(0, 0); + } if (!STATUS_SUCCESS.equals(task.getStatus()) && !STATUS_FAILED.equals(task.getStatus())) { // 任务仍在跑:补测只做终态之后的兜底,顺延不计数(避免与分片重提竞态) recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationService.java index a626d7e4f..6727ff146 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationService.java @@ -60,6 +60,10 @@ public class NotificationService { public static final String SCENE_SERVICE_DOWN = "service_down"; /** 麦象(18960 任务调度)异常:任务停滞/失败/队列积压。 */ public static final String SCENE_MAIXIANG_ANOMALY = "maixiang_anomaly"; + /** 代理/LLM 消耗异常,账号任务已被自动暂停。 */ + public static final String SCENE_COST_GUARD = "cost_guard"; + /** 同一把代理/LLM 密钥被过多账号共用。 */ + public static final String SCENE_SECRET_SHARED = "secret_shared"; public static final String SCENE_SYSTEM = "system"; /** @@ -78,10 +82,12 @@ public class NotificationService { private static final Map SCENE_TO_CATEGORY = Map.of( SCENE_SERVICE_DOWN, CATEGORY_SYSTEM_ERROR, SCENE_MAIXIANG_ANOMALY, CATEGORY_SYSTEM_ERROR, + SCENE_COST_GUARD, CATEGORY_SYSTEM_ERROR, SCENE_TASK_FAILED, CATEGORY_TASK_ERROR, SCENE_TASK_PARTIAL, CATEGORY_TASK_ERROR, SCENE_SECRET_BALANCE, CATEGORY_CONFIG_ERROR, SCENE_SECRET_INVALID, CATEGORY_CONFIG_ERROR, + SCENE_SECRET_SHARED, CATEGORY_CONFIG_ERROR, SCENE_SYSTEM, CATEGORY_SYSTEM_NOTICE); private static final long MAX_PAGE_SIZE = 100L; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java index ff8e30da5..e1cb404bb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java @@ -58,6 +58,10 @@ public class PriceTrackLoopRunService { private final PriceTrackLoopRunMapper loopRunMapper; private final FileTaskMapper fileTaskMapper; private final ObjectMapper objectMapper; + + /** 损耗闸:字段注入避免改动手工 new 的 7 处测试构造器;null=未启用(单测)。 */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard; private final ZiniaoShopSwitchService ziniaoShopSwitchService; @Transactional @@ -104,6 +108,11 @@ public class PriceTrackLoopRunService { @Transactional public PriceTrackLoopRunDispatchVo dispatchNext(Long loopRunId, Long userId) { + if (taskGuard != null && taskGuard.isPaused(userId)) { + // 不把循环标成 STOPPED:暂停只停派发,恢复后同一条循环还能接着跑 + throw new BusinessException(com.nanri.aiimage.modules.usersecret.service + .UserTaskGuardService.PAUSED_MESSAGE); + } PriceTrackLoopRunEntity entity = getOwnedLoopRun(loopRunId, userId); reconcileWithTerminalChild(entity); if (isTerminal(entity.getStatus())) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java index 56981bb99..15d8d69ab 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.LlmKeyGate; import com.nanri.aiimage.common.retry.LlmRetryBackoff; import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.HttpClientPool; @@ -50,6 +51,10 @@ public class SimilarAsinLlmClient { private final ObjectMapper objectMapper; private final ExternalCallMetricsRecorder externalCallMetrics; private final UserSecretUsageService userSecretUsageService; + + /** 损耗闸:字段注入避免改动手工构造的 LLM 用例;null=未启用(单测)。 */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService taskGuard; private final UnreachableCooldown unreachableCooldown; private volatile RestClient sharedRestClient; @@ -97,11 +102,23 @@ public class SimilarAsinLlmClient { Exception lastFailure = null; int connectFailures = 0; for (int attempt = 1; attempt <= attempts; attempt++) { - try { + try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire(resolvedKey)) { String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt); unreachableCooldown.recordConnected(host); return content; } catch (Exception ex) { + if (ex instanceof com.nanri.aiimage.common.exception.TaskPausedException paused) { + // 暂停是账号级结论,重试不会变好;已完成的行走正常落库 + throw paused; + } + if (ex instanceof LlmKeyGate.QueueTimeoutException) { + // 本地 key 容量已满:重试只会继续排队,直接交给行级失败 + throw (LlmKeyGate.QueueTimeoutException) ex; + } + if (LlmKeyGate.isRateLimited(ex)) { + // 429 时把同 key 的下一次许可整体后移,避免所有调用一起再撞 + LlmKeyGate.penalizeRateLimited(resolvedKey, 2_000L); + } lastFailure = ex; if (UnreachableCooldown.isConnectFailure(ex)) { connectFailures++; @@ -344,6 +361,10 @@ public class SimilarAsinLlmClient { if (context == null) { return; } + if (taskGuard != null && taskGuard.isPaused(context.userId())) { + throw new com.nanri.aiimage.common.exception.TaskPausedException( + com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.PAUSED_MESSAGE); + } userSecretUsageService.record(context.userId(), UserSecretModule.SIMILAR_ASIN.key(), 1); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/UserCostGuardController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/UserCostGuardController.java new file mode 100644 index 000000000..b91b33d86 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/UserCostGuardController.java @@ -0,0 +1,44 @@ +package com.nanri.aiimage.modules.usersecret.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.security.AdminAuthSupport; +import com.nanri.aiimage.modules.usersecret.model.vo.CostGuardStatusVo; +import com.nanri.aiimage.modules.usersecret.service.CostGuardService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 桌面端损耗监控:身份一律取自 JWT,不接受前端传 uid。 + * 前端每 30 秒轮询 status;暂停时弹窗,用户排查后点「继续任务」调 resume。 + */ +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/user/cost-guard") +@Tag(name = "损耗监控(桌面端)", description = "代理/LLM 异常消耗的暂停状态与手动继续。") +public class UserCostGuardController { + + private final CostGuardService costGuardService; + private final AdminAuthSupport adminAuthSupport; + + @GetMapping("/status") + @Operation(summary = "读取当前账号的损耗监控状态") + public ApiResponse status(HttpServletRequest request) { + return ApiResponse.success(costGuardService.status(currentUserId(request))); + } + + @PostMapping("/resume") + @Operation(summary = "排查后继续任务", description = "只清暂停闸与损耗基线,不删除已完成任务和结果行。") + public ApiResponse resume(HttpServletRequest request) { + return ApiResponse.success("已继续任务", costGuardService.resume(currentUserId(request))); + } + + private Long currentUserId(HttpServletRequest request) { + return adminAuthSupport.requireUser(request).getId(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java index 2700f9119..b4fc78547 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java @@ -1,11 +1,14 @@ package com.nanri.aiimage.modules.usersecret.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate; import com.nanri.aiimage.modules.usersecret.model.entity.UserSecretUsageEntity; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import java.time.LocalDate; +import java.util.List; @Mapper public interface UserSecretUsageMapper extends BaseMapper { @@ -33,6 +36,35 @@ public interface UserSecretUsageMapper extends BaseMapper @Param("promptTokens") long promptTokens, @Param("completionTokens") long completionTokens); + /** + * 当日用量按用户 × 模块聚合(损耗扫描每分钟读一次)。 + * 表是「用户 × 模块 × 天」的日聚合,行数 = 用户数 × 3,全量 GROUP BY 成本可忽略。 + */ + @Select(""" + SELECT user_id AS userId, + module_key AS moduleKey, + COALESCE(SUM(call_count), 0) AS callCount, + COALESCE(SUM(prompt_tokens + completion_tokens), 0) AS tokens + FROM biz_user_secret_usage_daily + WHERE business_date = #{businessDate} + GROUP BY user_id, module_key + """) + List sumByUserAndModule(@Param("businessDate") LocalDate businessDate); + + /** 单个用户的当日读数:客户端状态接口每 30 秒轮询一次,不能为此全表 GROUP BY。 */ + @Select(""" + SELECT user_id AS userId, + module_key AS moduleKey, + COALESCE(SUM(call_count), 0) AS callCount, + COALESCE(SUM(prompt_tokens + completion_tokens), 0) AS tokens + FROM biz_user_secret_usage_daily + WHERE business_date = #{businessDate} + AND user_id = #{userId} + GROUP BY user_id, module_key + """) + List sumForUser(@Param("userId") Long userId, + @Param("businessDate") LocalDate businessDate); + /** 分批删除保留期外的用量日统计行(该表此前只增不删)。 */ int deleteBefore(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserTaskGuardMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserTaskGuardMapper.java new file mode 100644 index 000000000..35edc7fe0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserTaskGuardMapper.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.usersecret.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Update; + +@Mapper +public interface UserTaskGuardMapper extends BaseMapper { + + /** 幂等暂停:无记录则建,有记录则刷新原因与时刻。 */ + @Insert(""" + INSERT INTO biz_user_task_guard (user_id, paused, reason, paused_at, updated_at) + VALUES (#{userId}, 1, #{reason}, NOW(), NOW()) + ON DUPLICATE KEY UPDATE paused = 1, reason = #{reason}, + paused_at = NOW(), updated_at = NOW() + """) + int pause(@Param("userId") Long userId, @Param("reason") String reason); + + @Update(""" + UPDATE biz_user_task_guard + SET paused = 0, reason = '', paused_at = NULL, updated_at = NOW() + WHERE user_id = #{userId} + """) + int resume(@Param("userId") Long userId); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageAggregate.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageAggregate.java new file mode 100644 index 000000000..5e31aea0f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageAggregate.java @@ -0,0 +1,13 @@ +package com.nanri.aiimage.modules.usersecret.model.dto; + +import lombok.Data; + +/** 按用户 × 模块聚合的当日用量读数(CostGuardScheduler 的扫描输入)。 */ +@Data +public class UserSecretUsageAggregate { + + private Long userId; + private String moduleKey; + private Long callCount; + private Long tokens; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserTaskGuardEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserTaskGuardEntity.java new file mode 100644 index 000000000..c8c62a480 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserTaskGuardEntity.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.usersecret.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** 账号级任务暂停闸:损耗异常时停新派发,排查后由用户手动恢复。 */ +@Data +@TableName("biz_user_task_guard") +public class UserTaskGuardEntity { + + @TableId(value = "user_id", type = IdType.INPUT) + private Long userId; + private Boolean paused; + private String reason; + private LocalDateTime pausedAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/CostGuardStatusVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/CostGuardStatusVo.java new file mode 100644 index 000000000..e46368fae --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/CostGuardStatusVo.java @@ -0,0 +1,49 @@ +package com.nanri.aiimage.modules.usersecret.model.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.time.LocalDateTime; +import java.util.List; + +/** 客户端「损耗监控」状态:暂停闸 + 当日用量 + 本轮增量 + 共享密钥提示。 */ +@Data +public class CostGuardStatusVo { + + private boolean paused; + private String reason; + private LocalDateTime pausedAt; + private TodayUsage today; + private WindowUsage window; + private Thresholds thresholds; + private List sharedWarnings; + + @Data + @AllArgsConstructor + public static class TodayUsage { + private long proxy; + private long llmCalls; + private long llmTokens; + } + + @Data + @AllArgsConstructor + public static class WindowUsage { + private int seconds; + private long proxy; + private long llmCalls; + private long llmTokens; + } + + @Data + @AllArgsConstructor + public static class Thresholds { + private int proxyWindow; + private int llmCallsWindow; + private long llmTokensWindow; + private int proxyDaily; + private int llmCallsDaily; + private long llmTokensDaily; + private int sharedAccounts; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/SharedSecretWarningVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/SharedSecretWarningVo.java new file mode 100644 index 000000000..297c061fb --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/vo/SharedSecretWarningVo.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.usersecret.model.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** 一把代理/LLM 密钥被多个账号共用的提示(只报账号数,不暴露其他账号身份)。 */ +@Data +@AllArgsConstructor +public class SharedSecretWarningVo { + + /** proxy / llm */ + private String type; + private String moduleLabel; + private int accounts; + private int threshold; + private String message; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardScheduler.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardScheduler.java new file mode 100644 index 000000000..f7e655534 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardScheduler.java @@ -0,0 +1,225 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.CostGuardProperties; +import com.nanri.aiimage.modules.notification.service.NotificationDispatchService; +import com.nanri.aiimage.modules.notification.service.NotificationService; +import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper; +import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate; +import com.nanri.aiimage.modules.usersecret.support.UserSecretModule; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.LocalDate; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 代理 / LLM 损耗扫描:每分钟读一次当日用量聚合表,用「本轮增量」判失控、用「日累计跨阈值」 + * 判慢烧;命中即置账号暂停闸并推用户通知。 + * + *

为什么用日聚合表的增量而不是另建明细表:代理计次本来就是 16890/pt_worker 按批上报的, + * LLM 计次在发请求前已落库;再建一条实时流水只会多一份可能对不上的账。日表行数 = 用户数×3, + * 每分钟全量 GROUP BY 的成本可以忽略。 + * + *

只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)}, + * 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class CostGuardScheduler { + + static final String LOCK_NAME = "cost-guard-scan"; + + private final CostGuardProperties properties; + private final UserSecretUsageMapper usageMapper; + private final UserTaskGuardService guardService; + private final SharedSecretMonitorService sharedSecretMonitorService; + private final NotificationDispatchService notificationDispatchService; + private final DistributedJobLockService distributedJobLockService; + + private final Map previousTotals = new ConcurrentHashMap<>(); + private final Map lastDeltas = new ConcurrentHashMap<>(); + private volatile long lastSharedRefreshAt = 0L; + + @Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}", + initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}") + public void scan() { + if (!properties.isEnabled()) { + return; + } + var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2)); + if (lock == null) { + log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮"); + return; + } + try (lock) { + runOnce(); + } catch (Exception ex) { + log.warn("[cost-guard] 损耗扫描异常终止 err={}", ex.getMessage(), ex); + } + } + + /** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */ + void runOnce() { + Map current = loadTodayTotals(); + Map deltas = new LinkedHashMap<>(); + int paused = 0; + for (Map.Entry entry : current.entrySet()) { + Long userId = entry.getKey(); + UserTotals now = entry.getValue(); + UserTotals before = previousTotals.get(userId); + if (before == null) { + // 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次 + deltas.put(userId, WindowDelta.ZERO); + continue; + } + WindowDelta delta = new WindowDelta( + Math.max(0, now.proxy - before.proxy), + Math.max(0, now.llmCalls - before.llmCalls), + Math.max(0, now.llmTokens - before.llmTokens)); + deltas.put(userId, delta); + if (guardService.isPaused(userId)) { + continue; + } + String reason = breachReason(now, before, delta); + if (reason == null) { + continue; + } + guardService.pause(userId, reason); + notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD, + NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason + + "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。", + null); + paused++; + log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason); + } + previousTotals.clear(); + previousTotals.putAll(current); + lastDeltas.clear(); + lastDeltas.putAll(deltas); + maybeRefreshShared(); + log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms", + current.size(), paused, sharedSecretMonitorService.snapshotAtMillis()); + } + + /** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */ + public void resetBaseline(Long userId) { + if (userId == null) { + return; + } + previousTotals.remove(userId); + lastDeltas.remove(userId); + } + + public WindowDelta lastDelta(Long userId) { + WindowDelta delta = userId == null ? null : lastDeltas.get(userId); + return delta == null ? WindowDelta.ZERO : delta; + } + + public Map loadTodayTotals() { + List rows = usageMapper.sumByUserAndModule(LocalDate.now()); + Map totals = new LinkedHashMap<>(); + for (UserSecretUsageAggregate row : rows) { + if (row.getUserId() == null || row.getUserId() <= 0) { + continue; + } + accumulate(totals.computeIfAbsent(row.getUserId(), key -> new UserTotals()), row); + } + return totals; + } + + private void accumulate(UserTotals item, UserSecretUsageAggregate row) { + long calls = row.getCallCount() == null ? 0L : row.getCallCount(); + long tokens = row.getTokens() == null ? 0L : row.getTokens(); + if (UserSecretModule.PROXY.key().equals(row.getModuleKey())) { + item.proxy += calls; + } else { + item.llmCalls += calls; + item.llmTokens += tokens; + } + } + + public UserTotals todayTotals(Long userId) { + if (userId == null || userId <= 0) { + return new UserTotals(); + } + try { + UserTotals totals = new UserTotals(); + for (UserSecretUsageAggregate row : usageMapper.sumForUser(userId, LocalDate.now())) { + accumulate(totals, row); + } + return totals; + } catch (Exception ex) { + // 状态接口是客户端每 30 秒一次的轮询:读不到就返回零值,不让一次库抖动打断提示 + log.warn("[cost-guard] 读取用户当日用量失败 userId={} err={}", userId, ex.getMessage()); + return new UserTotals(); + } + } + + private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) { + int seconds = Math.max(60, properties.getWindowSeconds()); + if (properties.getProxyWindowThreshold() > 0 + && delta.proxy >= properties.getProxyWindowThreshold()) { + return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 " + + properties.getProxyWindowThreshold() + " 次"; + } + if (properties.getLlmWindowCallsThreshold() > 0 + && delta.llmCalls >= properties.getLlmWindowCallsThreshold()) { + return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 " + + properties.getLlmWindowCallsThreshold() + " 次"; + } + if (properties.getLlmWindowTokensThreshold() > 0 + && delta.llmTokens >= properties.getLlmWindowTokensThreshold()) { + return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 " + + properties.getLlmWindowTokensThreshold(); + } + if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) { + return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 " + + properties.getDailyProxyThreshold() + " 次"; + } + if (crossed(before.llmCalls, now.llmCalls, properties.getDailyLlmCallsThreshold())) { + return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 " + + properties.getDailyLlmCallsThreshold() + " 次"; + } + if (crossed(before.llmTokens, now.llmTokens, properties.getDailyLlmTokensThreshold())) { + return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 " + + properties.getDailyLlmTokensThreshold(); + } + return null; + } + + private boolean crossed(long before, long now, long threshold) { + return threshold > 0 && before < threshold && now >= threshold; + } + + private void maybeRefreshShared() { + long now = System.currentTimeMillis(); + if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) { + return; + } + lastSharedRefreshAt = now; + try { + sharedSecretMonitorService.refresh(); + } catch (Exception ex) { + // 共享密钥扫描失败不影响损耗暂停主流程,下一轮再试 + log.warn("[cost-guard] 共享密钥扫描失败 err={}", ex.getMessage(), ex); + } + } + + public static final class UserTotals { + public long proxy; + public long llmCalls; + public long llmTokens; + } + + public record WindowDelta(long proxy, long llmCalls, long llmTokens) { + public static final WindowDelta ZERO = new WindowDelta(0, 0, 0); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardService.java new file mode 100644 index 000000000..3a4591b4d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/CostGuardService.java @@ -0,0 +1,53 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.nanri.aiimage.config.CostGuardProperties; +import com.nanri.aiimage.modules.usersecret.model.vo.CostGuardStatusVo; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** 客户端损耗监控的状态读取与手动继续;不改任何任务结果行。 */ +@Service +@RequiredArgsConstructor +public class CostGuardService { + + private final CostGuardProperties properties; + private final UserTaskGuardService guardService; + private final CostGuardScheduler costGuardScheduler; + private final SharedSecretMonitorService sharedSecretMonitorService; + + public CostGuardStatusVo status(Long userId) { + UserTaskGuardService.GuardState state = guardService.state(userId); + CostGuardScheduler.UserTotals today = costGuardScheduler.todayTotals(userId); + CostGuardScheduler.WindowDelta delta = costGuardScheduler.lastDelta(userId); + int windowSeconds = Math.max(60, properties.getWindowSeconds()); + + CostGuardStatusVo vo = new CostGuardStatusVo(); + vo.setPaused(state.paused()); + vo.setReason(state.reason()); + vo.setPausedAt(state.pausedAt()); + vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens)); + vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds, + delta.proxy(), delta.llmCalls(), delta.llmTokens())); + vo.setThresholds(new CostGuardStatusVo.Thresholds( + properties.getProxyWindowThreshold(), + properties.getLlmWindowCallsThreshold(), + properties.getLlmWindowTokensThreshold(), + properties.getDailyProxyThreshold(), + properties.getDailyLlmCallsThreshold(), + properties.getDailyLlmTokensThreshold(), + properties.getSharedAccountsThreshold())); + List shared = + sharedSecretMonitorService.warningsFor(userId); + vo.setSharedWarnings(shared == null ? List.of() : shared); + return vo; + } + + /** 用户排查后手动继续:清暂停位并把损耗基线对齐到当前,已完成的任务与结果原样保留。 */ + public CostGuardStatusVo resume(Long userId) { + guardService.resume(userId); + costGuardScheduler.resetBaseline(userId); + return status(userId); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorService.java new file mode 100644 index 000000000..0d38bd094 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorService.java @@ -0,0 +1,150 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.nanri.aiimage.config.CostGuardProperties; +import com.nanri.aiimage.modules.notification.service.NotificationDispatchService; +import com.nanri.aiimage.modules.notification.service.NotificationService; +import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper; +import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity; +import com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo; +import com.nanri.aiimage.modules.usersecret.support.UserSecretModule; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 共享密钥提示:同一把代理/LLM 密钥被过多账号共用时,给每个使用者一条明确提示。 + * + *

只统计、不暂停:共用同一把密钥本身不是故障(账单按 key 计费,平台也允许这样配), + * 但它会把一次异常消耗放大到所有账号,也会让「谁烧的」无法归因。提示里只报账号数, + * 不暴露其他账号身份。 + * + *

分组键直接用密文:{@code ShopCredentialCryptoService} 是 AES/ECB 无 IV, + * 同明文必同密文(与 {@link UserApiSecretService#checkAllForScheduler} 同一口径), + * 因此本类不需要解密、也不会把明文密钥带进内存快照。 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class SharedSecretMonitorService { + + private final UserApiSecretMapper userApiSecretMapper; + private final CostGuardProperties properties; + private final NotificationDispatchService notificationDispatchService; + + private volatile Map> warningsByUser = Map.of(); + private volatile long snapshotAtMillis = 0L; + + /** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */ + public synchronized void refresh() { + if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) { + warningsByUser = Map.of(); + snapshotAtMillis = System.currentTimeMillis(); + return; + } + // 用字符串列名而不是 lambda:本类在纯单测里不经过 MyBatis 的实体元数据初始化 + List rows = userApiSecretMapper.selectList( + new QueryWrapper() + .select("user_id", "module_key", "secret_value")); + // groupKey = module|密文;值是该组的使用账号集合(去重) + Map groups = new LinkedHashMap<>(); + for (UserApiSecretEntity row : rows) { + if (row.getUserId() == null || row.getUserId() <= 0) { + continue; + } + Optional module = UserSecretModule.of(row.getModuleKey()); + if (module.isEmpty() || row.getSecretValue() == null || row.getSecretValue().isBlank()) { + continue; + } + String groupKey = row.getModuleKey() + '|' + row.getSecretValue(); + groups.computeIfAbsent(groupKey, key -> new SecretGroup(module.get())) + .userIds.add(row.getUserId()); + } + + Map> next = new LinkedHashMap<>(); + int hitGroups = 0; + for (Map.Entry entry : groups.entrySet()) { + SecretGroup group = entry.getValue(); + int accounts = group.userIds.size(); + if (accounts < properties.getSharedAccountsThreshold()) { + continue; + } + hitGroups++; + String type = group.module == UserSecretModule.PROXY ? "proxy" : "llm"; + int threshold = properties.getSharedAccountsThreshold(); + String message = buildMessage(group.module, accounts, threshold); + SharedSecretWarningVo warning = new SharedSecretWarningVo( + type, group.module.label(), accounts, threshold, message); + for (Long userId : group.userIds) { + next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning); + pushDailyWarning(userId, group.module, entry.getKey(), warning); + } + } + warningsByUser = next; + snapshotAtMillis = System.currentTimeMillis(); + log.info("[cost-guard] 共享密钥扫描完成 配置行={} 密钥组={} 超阈值组={} 受影响账号={}", + rows.size(), groups.size(), hitGroups, next.size()); + } + + /** 当前用户共享密钥提示;快照未建立时返回空,不阻塞客户端轮询。 */ + public List warningsFor(Long userId) { + if (userId == null || userId <= 0) { + return List.of(); + } + return warningsByUser.getOrDefault(userId, List.of()); + } + + public long snapshotAtMillis() { + return snapshotAtMillis; + } + + private void pushDailyWarning(Long userId, UserSecretModule module, String groupKey, + SharedSecretWarningVo warning) { + String day = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE); + String dedupe = "secret_shared:" + module.key() + ':' + sha256Hex(groupKey) + + ':' + userId + ':' + day; + notificationDispatchService.pushToUser(userId, NotificationService.SCENE_SECRET_SHARED, + NotificationService.LEVEL_WARNING, + module.label() + "被 " + warning.getAccounts() + " 个账号共用", + warning.getMessage(), dedupe); + } + + private String buildMessage(UserSecretModule module, int accounts, int threshold) { + String what = module == UserSecretModule.PROXY ? "代理提取链接" : "LLM 密钥"; + return "你的" + module.label() + "(" + what + ")当前被 " + accounts + + " 个账号共用,已达到提示阈值 " + threshold + "。任一账号的异常消耗都会算在这把密钥的套餐上," + + "建议在「密钥设置」里为不同账号拆分独立密钥,便于控费和定位问题。"; + } + + private String sha256Hex(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (Exception ex) { + return Integer.toHexString(value.hashCode()); + } + } + + private static final class SecretGroup { + private final UserSecretModule module; + private final Set userIds = new HashSet<>(); + + private SecretGroup(UserSecretModule module) { + this.module = module; + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardService.java new file mode 100644 index 000000000..10dbf4cfc --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardService.java @@ -0,0 +1,118 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.nanri.aiimage.modules.usersecret.mapper.UserTaskGuardMapper; +import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * 账号级任务暂停闸:损耗异常时由 CostGuardScheduler 置位,派发入口据此停新任务。 + * + *

读侧带 3 秒内存缓存:品牌检测/LLM 每次请求都问一次,直接打库会把闸门本身变成热点; + * 多实例下最迟 3 秒收敛,足够挡住下一分钟的任务。写侧立即清缓存,本机无延迟。 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class UserTaskGuardService { + + public static final String PAUSED_MESSAGE = + "账号任务已暂停(损耗保护),请排查代理/LLM 消耗后在客户端点击「继续任务」"; + + private static final long CACHE_TTL_NANOS = TimeUnit.SECONDS.toNanos(3); + private static final int REASON_MAX_LENGTH = 500; + + private final UserTaskGuardMapper guardMapper; + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + public boolean isPaused(Long userId) { + if (userId == null || userId <= 0) { + return false; + } + CachedState cached = cache.get(userId); + if (cached != null && System.nanoTime() - cached.readAtNanos() < CACHE_TTL_NANOS) { + return cached.paused(); + } + GuardState state = load(userId); + cache.put(userId, new CachedState(state.paused(), state.reason(), state.pausedAt(), + System.nanoTime())); + return state.paused(); + } + + /** 状态查询走库:客户端弹窗要看到其他实例刚写入的暂停,不吃本机缓存。 */ + public GuardState state(Long userId) { + if (userId == null || userId <= 0) { + return GuardState.ACTIVE; + } + GuardState state = load(userId); + cache.put(userId, new CachedState(state.paused(), state.reason(), state.pausedAt(), + System.nanoTime())); + return state; + } + + public void pause(Long userId, String reason) { + if (userId == null || userId <= 0) { + return; + } + String safeReason = truncate(reason); + try { + guardMapper.pause(userId, safeReason); + cache.remove(userId); + log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason); + } catch (Exception ex) { + // 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里 + log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}", + userId, safeReason, ex.getMessage(), ex); + } + } + + /** 用户手动继续:只清暂停位,不删任何已完成的任务/结果。 */ + public boolean resume(Long userId) { + if (userId == null || userId <= 0) { + return false; + } + try { + guardMapper.resume(userId); + cache.remove(userId); + log.info("[cost-guard] 用户手动继续任务 userId={}", userId); + return true; + } catch (Exception ex) { + log.error("[cost-guard] 继续任务写入失败 userId={} err={}", userId, ex.getMessage(), ex); + return false; + } + } + + private GuardState load(Long userId) { + try { + UserTaskGuardEntity row = guardMapper.selectById(userId); + if (row == null || !Boolean.TRUE.equals(row.getPaused())) { + return GuardState.ACTIVE; + } + return new GuardState(true, row.getReason() == null ? "" : row.getReason(), + row.getPausedAt()); + } catch (Exception ex) { + // 读不到闸门状态时按「不暂停」走:数据库抖动不该把全体用户的合法任务一起冻住。 + // 真正的失控由下一轮扫描重新置位。 + log.warn("[cost-guard] 读取暂停状态失败,按未暂停处理 userId={} err={}", + userId, ex.getMessage()); + return GuardState.ACTIVE; + } + } + + private String truncate(String reason) { + String value = reason == null ? "" : reason.trim(); + return value.length() <= REASON_MAX_LENGTH ? value : value.substring(0, REASON_MAX_LENGTH); + } + + private record CachedState(boolean paused, String reason, LocalDateTime pausedAt, long readAtNanos) { + } + + public record GuardState(boolean paused, String reason, LocalDateTime pausedAt) { + public static final GuardState ACTIVE = new GuardState(false, "", null); + } +} diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml index e9ed13065..c22541b59 100644 --- a/backend-java/src/main/resources/application-local.example.yml +++ b/backend-java/src/main/resources/application-local.example.yml @@ -9,6 +9,13 @@ AIIMAGE_INSTANCE_ID=local-121 AIIMAGE_SERVER_PORT=18080 AIIMAGE_INTERNAL_TOKEN=change-me-local-internal-token +# 代理/LLM 损耗保护:异常放大时暂停该账号新任务;共享密钥超过阈值只提示不暂停。 +AIIMAGE_COST_GUARD_ENABLED=true +AIIMAGE_COST_GUARD_PROXY_WINDOW_THRESHOLD=600 +AIIMAGE_COST_GUARD_LLM_WINDOW_CALLS_THRESHOLD=180 +AIIMAGE_COST_GUARD_LLM_WINDOW_TOKENS_THRESHOLD=600000 +AIIMAGE_COST_GUARD_SHARED_ACCOUNTS_THRESHOLD=3 + AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false AIIMAGE_DB_USERNAME=change-me AIIMAGE_DB_PASSWORD=change-me diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index b942310a6..914231012 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -501,6 +501,21 @@ aiimage: archive-connect-timeout-millis: ${AIIMAGE_IMAGE_VIDEO_ARCHIVE_CONNECT_TIMEOUT_MILLIS:10000} archive-read-timeout-millis: ${AIIMAGE_IMAGE_VIDEO_ARCHIVE_READ_TIMEOUT_MILLIS:600000} archive-max-attempts: ${AIIMAGE_IMAGE_VIDEO_ARCHIVE_MAX_ATTEMPTS:3} + # 代理/LLM 损耗保护:分钟增量与当日累计双阈值,命中即暂停该账号新任务; + # 共享密钥阈值只提示不暂停(共用本身不是故障,但会把异常消耗放大到所有账号)。 + cost-guard: + enabled: ${AIIMAGE_COST_GUARD_ENABLED:true} + scan-interval-ms: ${AIIMAGE_COST_GUARD_SCAN_INTERVAL_MS:60000} + scan-initial-delay-ms: ${AIIMAGE_COST_GUARD_SCAN_INITIAL_DELAY_MS:30000} + window-seconds: ${AIIMAGE_COST_GUARD_WINDOW_SECONDS:60} + proxy-window-threshold: ${AIIMAGE_COST_GUARD_PROXY_WINDOW_THRESHOLD:600} + llm-window-calls-threshold: ${AIIMAGE_COST_GUARD_LLM_WINDOW_CALLS_THRESHOLD:180} + llm-window-tokens-threshold: ${AIIMAGE_COST_GUARD_LLM_WINDOW_TOKENS_THRESHOLD:600000} + daily-proxy-threshold: ${AIIMAGE_COST_GUARD_DAILY_PROXY_THRESHOLD:8000} + daily-llm-calls-threshold: ${AIIMAGE_COST_GUARD_DAILY_LLM_CALLS_THRESHOLD:6000} + daily-llm-tokens-threshold: ${AIIMAGE_COST_GUARD_DAILY_LLM_TOKENS_THRESHOLD:20000000} + shared-accounts-threshold: ${AIIMAGE_COST_GUARD_SHARED_ACCOUNTS_THRESHOLD:3} + shared-scan-interval-ms: ${AIIMAGE_COST_GUARD_SHARED_SCAN_INTERVAL_MS:600000} user-secret: check-enabled: ${AIIMAGE_USER_SECRET_CHECK_ENABLED:true} check-cron: ${AIIMAGE_USER_SECRET_CHECK_CRON:0 30 4 * * *} diff --git a/backend-java/src/main/resources/db/V170__user_task_guard.sql b/backend-java/src/main/resources/db/V170__user_task_guard.sql new file mode 100644 index 000000000..aa7ed97fb --- /dev/null +++ b/backend-java/src/main/resources/db/V170__user_task_guard.sql @@ -0,0 +1,30 @@ +-- ============================================================= +-- 迁移 V168__user_task_guard +-- +-- 1. 变更目的:新增「账号级任务暂停闸」。 +-- - 代理提取 / LLM 调用在短时间内异常放大(或同一把密钥被过多账号共用)时, +-- 由 CostGuardScheduler 自动置 paused=1,所有派发入口据此停新任务; +-- - 用户在客户端排查后点「继续任务」,走 resume 接口置 paused=0。 +-- - 暂停只停新派发,不删除、不回滚已完成的任务与结果行。 +-- +-- 2. 口径: +-- - 一行一个 user_id(主键),无记录 = 未暂停; +-- - reason 存给用户看的暂停原因(含触发数值),最长 512; +-- - paused_at 只记录最后一次自动暂停时刻,恢复时清空。 +-- +-- 3. 影响表与数据量:新表 1 张,空表,无存量数据迁移。 +-- +-- 4. 【可重跑性】CREATE TABLE IF NOT EXISTS,重跑 no-op。 +-- +-- 5. 回滚(如需):DROP TABLE IF EXISTS `biz_user_task_guard`; +-- ============================================================= + +CREATE TABLE IF NOT EXISTS `biz_user_task_guard` ( + `user_id` BIGINT NOT NULL COMMENT '账号 id', + `paused` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '1=暂停新任务派发', + `reason` VARCHAR(512) NOT NULL DEFAULT '' COMMENT '暂停原因(展示给用户)', + `paused_at` DATETIME NULL COMMENT '最后一次自动暂停时刻', + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`user_id`), + KEY `idx_user_task_guard_paused` (`paused`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='账号级任务暂停闸(损耗保护)'; diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmKeyGateTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmKeyGateTest.java new file mode 100644 index 000000000..517ffed2d --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmKeyGateTest.java @@ -0,0 +1,38 @@ +package com.nanri.aiimage.common.retry; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; + +import static org.assertj.core.api.Assertions.assertThat; + +class LlmKeyGateTest { + + @AfterEach + void tearDown() { + LlmKeyGate.resetForTests(); + } + + @Test + void recognizesRateLimitThroughExceptionChain() { + assertThat(LlmKeyGate.isRateLimited( + new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS))).isTrue(); + assertThat(LlmKeyGate.isRateLimited( + new IllegalStateException("wrapped", + new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)))).isTrue(); + assertThat(LlmKeyGate.isRateLimited( + new IllegalStateException("read timeout"))).isFalse(); + } + + @Test + void permitIsReleasedAfterUse() { + try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire("shared-key")) { + assertThat(ignored).isNotNull(); + } + // 能再次拿到同一个 key 的许可,说明 close 归还了信号量 + try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire("shared-key")) { + assertThat(ignored).isNotNull(); + } + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java index 1b6bd16ad..1d46cf2ca 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java @@ -81,6 +81,31 @@ class BrandCheckClientTest { } } + @Test + void pausedAccountStopsBatchWithoutHittingUpstream() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + HttpServer server = startBrandCheckServer(exchange -> { + attempts.incrementAndGet(); + respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[]}"); + }); + try { + BrandCheckClient client = newClient(server, 3); + com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService guard = + mock(com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.class); + when(guard.isPaused(7L)).thenReturn(true); + ReflectionTestUtils.setField(client, "taskGuard", guard); + + BrandCheckClient.BrandCheckBatchResult result = + client.checkAll(List.of("Apple", "Nike"), "Terms", 7L); + + assertThat(attempts.get()).as("账号已暂停时一个上游请求都不该发").isZero(); + assertThat(result.queryFaildData()).containsExactly("Apple", "Nike"); + assertThat(result.reason()).contains("账号任务已暂停"); + } finally { + server.stop(0); + } + } + @Test void queryFailureRetriesUntilSuccess() throws Exception { AtomicInteger attempts = new AtomicInteger(); @@ -176,6 +201,60 @@ class BrandCheckClientTest { } } + /** + * 出口被上游连续拒绝时,整批不再给每个品牌各发一次首轮探针。 + * + *

2026-10-23 账单形状:16890 每个品牌最多换 2 个出口 IP,Java 再按品牌重试 + * 10 轮;一批几千个品牌就是几千×2 次提取。出口熔断证据只需前 2 个品牌, + * 后面的品牌直接判查询失败转补测。 + */ + @Test + void egressClassifierMatchesUpstreamReason() { + assertThat(com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice.isEgressRejected( + "上游拒绝代理出口 IP,换 IP 重试仍未放行")).isTrue(); + } + + @Test + void egressReasonParsesFromUpstream() throws Exception { + HttpServer server = startBrandCheckServer(exchange -> respond(exchange, + "{\"faild_data\":[],\"query_faild_data\":[\"x\"]," + + "\"reason\":\"上游拒绝代理出口 IP,换 IP 重试仍未放行\",\"retryable\":false}")); + try { + BrandCheckClient client = newClient(server, 1); + BrandCheckClient.BrandCheckResponse response = client.check("b1", "Terms", 7L); + assertThat(response.getReason()).contains("代理出口"); + assertThat(response.isRetryable()).isFalse(); + } finally { + server.stop(0); + } + } + + @Test + void egressRejectionStopsBatchAfterTwoBrands() throws Exception { + AtomicInteger requests = new AtomicInteger(); + HttpServer server = startBrandCheckServer(exchange -> { + requests.incrementAndGet(); + respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[\"x\"]," + + "\"reason\":\"上游拒绝代理出口 IP,换 IP 重试仍未放行\",\"retryable\":false}"); + }); + try { + BrandCheckClient client = newClient(server, 10); + List brands = List.of("b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8"); + + BrandCheckClient.BrandCheckBatchResult result = client.checkAll(brands, "Terms", 7L); + + assertThat(requests.get()) + // 检查池并发 3:三条在飞 + 一条在首个响应回来后替补进来,最多 4 次; + // 关键是这个上界与品牌数无关,不能回到「品牌数 × 10 轮」的放大形态 + .as("出口被拒后只允许并发在飞的前几个品牌各打一次,不能按品牌数×10 轮放大") + .isBetween(1, 4); + assertThat(result.hasQueryFailedData()).isTrue(); + assertThat(result.reason()).contains("代理出口"); + } finally { + server.stop(0); + } + } + /** * 短路判据必须保守:只有"跑满整条阶梯仍每轮查询失败"才成立。 * 成功品牌首轮就返回、从不置标志,所以同一批里的个别失败品牌要照旧拿满它的重试机会—— diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunServiceTest.java index 7992c42c7..a6d6580d2 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunServiceTest.java @@ -31,6 +31,24 @@ import static org.mockito.Mockito.when; class PriceTrackLoopRunServiceTest { + @Test + void pausedAccountCannotDispatchNext() { + PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class); + PriceTrackLoopRunService service = new PriceTrackLoopRunService( + loopRunMapper, mock(FileTaskMapper.class), new ObjectMapper(), + mock(ZiniaoShopSwitchService.class)); + com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService guard = + mock(com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.class); + when(guard.isPaused(1L)).thenReturn(true); + org.springframework.test.util.ReflectionTestUtils.setField(service, "taskGuard", guard); + + BusinessException error = assertThrows(BusinessException.class, + () -> service.dispatchNext(743L, 1L)); + + assertTrue(error.getMessage().contains("账号任务已暂停")); + verify(loopRunMapper, never()).selectById(743L); + } + @Test void syncLoopRunAfterChildTerminalUsesChildContextWhenLoopRowAlreadyDrifted() throws Exception { ObjectMapper objectMapper = new ObjectMapper(); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/CostGuardSchedulerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/CostGuardSchedulerTest.java new file mode 100644 index 000000000..a69eab1b2 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/CostGuardSchedulerTest.java @@ -0,0 +1,137 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.CostGuardProperties; +import com.nanri.aiimage.modules.notification.service.NotificationDispatchService; +import com.nanri.aiimage.modules.notification.service.NotificationService; +import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper; +import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDate; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class CostGuardSchedulerTest { + + @Mock + private UserSecretUsageMapper usageMapper; + @Mock + private UserTaskGuardService guardService; + @Mock + private SharedSecretMonitorService sharedSecretMonitorService; + @Mock + private NotificationDispatchService notificationDispatchService; + @Mock + private DistributedJobLockService distributedJobLockService; + + private CostGuardScheduler scheduler; + + @BeforeEach + void setUp() { + CostGuardProperties properties = new CostGuardProperties(); + properties.setProxyWindowThreshold(10); + properties.setLlmWindowCallsThreshold(5); + properties.setLlmWindowTokensThreshold(1_000); + properties.setDailyProxyThreshold(100); + properties.setDailyLlmCallsThreshold(100); + properties.setDailyLlmTokensThreshold(10_000); + scheduler = new CostGuardScheduler(properties, usageMapper, guardService, + sharedSecretMonitorService, notificationDispatchService, distributedJobLockService); + } + + @Test + void firstScanOnlyBuildsBaseline() { + when(usageMapper.sumByUserAndModule(any(LocalDate.class))) + .thenReturn(List.of(row(7L, "proxy", 50, 0))); + when(usageMapper.sumForUser(eq(7L), any(LocalDate.class))) + .thenReturn(List.of(row(7L, "proxy", 50, 0))); + + scheduler.runOnce(); + + verify(guardService, never()).pause(anyLong(), anyString()); + assertThat(scheduler.todayTotals(7L).proxy).isEqualTo(50); + } + + @Test + void proxyWindowJumpPausesAndNotifies() { + when(usageMapper.sumByUserAndModule(any(LocalDate.class))) + .thenReturn(List.of(row(7L, "proxy", 0, 0))) + .thenReturn(List.of(row(7L, "proxy", 12, 0))); + when(guardService.isPaused(7L)).thenReturn(false); + + scheduler.runOnce(); + scheduler.runOnce(); + + verify(guardService).pause(eq(7L), contains("代理提取 12 次")); + verify(notificationDispatchService).pushToUser(eq(7L), + eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR), + anyString(), contains("继续任务"), isNull()); + } + + @Test + void llmCallsWindowJumpPauses() { + when(usageMapper.sumByUserAndModule(any(LocalDate.class))) + .thenReturn(List.of(row(8L, "similar-asin", 0, 0))) + .thenReturn(List.of(row(8L, "similar-asin", 6, 100))); + when(guardService.isPaused(8L)).thenReturn(false); + + scheduler.runOnce(); + scheduler.runOnce(); + + verify(guardService).pause(eq(8L), contains("LLM 调用 6 次")); + } + + @Test + void dailyCrossingOnlyPausesOnceAndRespectsResetBaseline() { + when(usageMapper.sumByUserAndModule(any(LocalDate.class))) + .thenReturn(List.of(row(9L, "proxy", 99, 0))) + .thenReturn(List.of(row(9L, "proxy", 100, 0))) + .thenReturn(List.of(row(9L, "proxy", 101, 0))); + when(guardService.isPaused(9L)).thenReturn(false); + + scheduler.runOnce(); + scheduler.runOnce(); + scheduler.resetBaseline(9L); + scheduler.runOnce(); + + verify(guardService).pause(eq(9L), contains("当日上限 100 次")); + } + + @Test + void alreadyPausedUserIsNotPausedAgain() { + when(usageMapper.sumByUserAndModule(any(LocalDate.class))) + .thenReturn(List.of(row(7L, "proxy", 0, 0))) + .thenReturn(List.of(row(7L, "proxy", 50, 0))); + when(guardService.isPaused(7L)).thenReturn(true); + + scheduler.runOnce(); + scheduler.runOnce(); + + verify(guardService, never()).pause(anyLong(), anyString()); + } + + private UserSecretUsageAggregate row(long userId, String module, long calls, long tokens) { + UserSecretUsageAggregate row = new UserSecretUsageAggregate(); + row.setUserId(userId); + row.setModuleKey(module); + row.setCallCount(calls); + row.setTokens(tokens); + return row; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorServiceTest.java new file mode 100644 index 000000000..783036745 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/SharedSecretMonitorServiceTest.java @@ -0,0 +1,101 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.nanri.aiimage.config.CostGuardProperties; +import com.nanri.aiimage.modules.notification.service.NotificationDispatchService; +import com.nanri.aiimage.modules.notification.service.NotificationService; +import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper; +import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity; +import com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SharedSecretMonitorServiceTest { + + @Mock + private UserApiSecretMapper userApiSecretMapper; + @Mock + private NotificationDispatchService notificationDispatchService; + + private CostGuardProperties properties; + private SharedSecretMonitorService service; + + @BeforeEach + void setUp() { + properties = new CostGuardProperties(); + properties.setSharedAccountsThreshold(3); + service = new SharedSecretMonitorService(userApiSecretMapper, properties, + notificationDispatchService); + } + + @Test + void threeAccountsSharingProxyProducesWarningAndDailyNotice() { + when(userApiSecretMapper.selectList(any())).thenReturn(List.of( + row(1L, "proxy", "same-cipher"), + row(2L, "proxy", "same-cipher"), + row(3L, "proxy", "same-cipher"), + row(4L, "proxy", "own-cipher"))); + + service.refresh(); + + List warnings = service.warningsFor(1L); + assertThat(warnings).hasSize(1); + assertThat(warnings.get(0).getType()).isEqualTo("proxy"); + assertThat(warnings.get(0).getAccounts()).isEqualTo(3); + assertThat(warnings.get(0).getMessage()).contains("3 个账号共用").contains("拆分独立密钥"); + assertThat(service.warningsFor(4L)).isEmpty(); + verify(notificationDispatchService, times(3)).pushToUser(anyLong(), + eq(NotificationService.SCENE_SECRET_SHARED), eq(NotificationService.LEVEL_WARNING), + contains("3 个账号共用"), contains("拆分独立密钥"), anyString()); + } + + @Test + void belowThresholdProducesNoWarning() { + when(userApiSecretMapper.selectList(any())).thenReturn(List.of( + row(1L, "similar-asin", "same"), + row(2L, "similar-asin", "same"))); + + service.refresh(); + + assertThat(service.warningsFor(1L)).isEmpty(); + verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(), + anyString(), anyString(), anyString()); + } + + @Test + void llmModuleIsTypedAsLlm() { + when(userApiSecretMapper.selectList(any())).thenReturn(List.of( + row(1L, "appearance-patent", "llm-key"), + row(2L, "appearance-patent", "llm-key"), + row(3L, "appearance-patent", "llm-key"))); + + service.refresh(); + + assertThat(service.warningsFor(2L).get(0).getType()).isEqualTo("llm"); + assertThat(service.warningsFor(2L).get(0).getModuleLabel()).isEqualTo("外观专利密钥"); + } + + private UserApiSecretEntity row(long userId, String moduleKey, String cipher) { + UserApiSecretEntity row = new UserApiSecretEntity(); + row.setUserId(userId); + row.setModuleKey(moduleKey); + row.setSecretValue(cipher); + return row; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardServiceTest.java new file mode 100644 index 000000000..28c0d7f76 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/usersecret/service/UserTaskGuardServiceTest.java @@ -0,0 +1,66 @@ +package com.nanri.aiimage.modules.usersecret.service; + +import com.nanri.aiimage.modules.usersecret.mapper.UserTaskGuardMapper; +import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class UserTaskGuardServiceTest { + + @Mock + private UserTaskGuardMapper guardMapper; + + @Test + void pausedStateIsCachedForHotPaths() { + UserTaskGuardEntity row = new UserTaskGuardEntity(); + row.setUserId(7L); + row.setPaused(true); + row.setReason("代理提取异常"); + row.setPausedAt(LocalDateTime.now()); + when(guardMapper.selectById(7L)).thenReturn(row); + UserTaskGuardService service = new UserTaskGuardService(guardMapper); + + assertThat(service.isPaused(7L)).isTrue(); + assertThat(service.isPaused(7L)).isTrue(); + + verify(guardMapper, org.mockito.Mockito.times(1)).selectById(7L); + } + + @Test + void pauseAndResumeClearCacheImmediately() { + UserTaskGuardService service = new UserTaskGuardService(guardMapper); + when(guardMapper.selectById(7L)).thenReturn(null); + + service.pause(7L, "LLM 消耗异常"); + verify(guardMapper).pause(7L, "LLM 消耗异常"); + + service.resume(7L); + verify(guardMapper).resume(7L); + assertThat(service.isPaused(7L)).isFalse(); + } + + @Test + void missingUserIsNeverPaused() { + UserTaskGuardService service = new UserTaskGuardService(guardMapper); + + assertThat(service.isPaused(null)).isFalse(); + assertThat(service.isPaused(0L)).isFalse(); + } + + @Test + void mapperFailureFallsBackToNotPaused() { + when(guardMapper.selectById(7L)).thenThrow(new IllegalStateException("db down")); + UserTaskGuardService service = new UserTaskGuardService(guardMapper); + + assertThat(service.isPaused(7L)).isFalse(); + } +}