feat(cost-guard): 代理/LLM 损耗监控与账号级任务暂停

- 每分钟扫描用量增量/日累计,异常置暂停闸并推站内通知
- 品牌补测、跟价派发、外观激活、品牌检测与两个 LLM 客户端统一接闸
- 同一代理/LLM 密钥被 3 个以上账号共用时给出用户提示
- 品牌检测出口拒绝批级熔断;LLM 同 key 共享并发/间隔/429 惩罚
- V170 新增 biz_user_task_guard
This commit is contained in:
2026-10-24 10:50:20 +08:00
parent 052f2bd4a5
commit 9c4905c94e
32 changed files with 1625 additions and 5 deletions
@@ -0,0 +1,38 @@
package com.nanri.aiimage.common.retry;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.springframework.http.HttpStatus;
import org.springframework.web.client.HttpClientErrorException;
import static org.assertj.core.api.Assertions.assertThat;
class LlmKeyGateTest {
@AfterEach
void tearDown() {
LlmKeyGate.resetForTests();
}
@Test
void recognizesRateLimitThroughExceptionChain() {
assertThat(LlmKeyGate.isRateLimited(
new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS))).isTrue();
assertThat(LlmKeyGate.isRateLimited(
new IllegalStateException("wrapped",
new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)))).isTrue();
assertThat(LlmKeyGate.isRateLimited(
new IllegalStateException("read timeout"))).isFalse();
}
@Test
void permitIsReleasedAfterUse() {
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire("shared-key")) {
assertThat(ignored).isNotNull();
}
// 能再次拿到同一个 key 的许可,说明 close 归还了信号量
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire("shared-key")) {
assertThat(ignored).isNotNull();
}
}
}
@@ -81,6 +81,31 @@ class BrandCheckClientTest {
}
}
@Test
void pausedAccountStopsBatchWithoutHittingUpstream() throws Exception {
AtomicInteger attempts = new AtomicInteger();
HttpServer server = startBrandCheckServer(exchange -> {
attempts.incrementAndGet();
respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[]}");
});
try {
BrandCheckClient client = newClient(server, 3);
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService guard =
mock(com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.class);
when(guard.isPaused(7L)).thenReturn(true);
ReflectionTestUtils.setField(client, "taskGuard", guard);
BrandCheckClient.BrandCheckBatchResult result =
client.checkAll(List.of("Apple", "Nike"), "Terms", 7L);
assertThat(attempts.get()).as("账号已暂停时一个上游请求都不该发").isZero();
assertThat(result.queryFaildData()).containsExactly("Apple", "Nike");
assertThat(result.reason()).contains("账号任务已暂停");
} finally {
server.stop(0);
}
}
@Test
void queryFailureRetriesUntilSuccess() throws Exception {
AtomicInteger attempts = new AtomicInteger();
@@ -176,6 +201,60 @@ class BrandCheckClientTest {
}
}
/**
* 出口被上游连续拒绝时,整批不再给每个品牌各发一次首轮探针。
*
* <p>2026-10-23 账单形状:16890 每个品牌最多换 2 个出口 IP,Java 再按品牌重试
* 10 轮;一批几千个品牌就是几千×2 次提取。出口熔断证据只需前 2 个品牌,
* 后面的品牌直接判查询失败转补测。
*/
@Test
void egressClassifierMatchesUpstreamReason() {
assertThat(com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice.isEgressRejected(
"上游拒绝代理出口 IP,换 IP 重试仍未放行")).isTrue();
}
@Test
void egressReasonParsesFromUpstream() throws Exception {
HttpServer server = startBrandCheckServer(exchange -> respond(exchange,
"{\"faild_data\":[],\"query_faild_data\":[\"x\"],"
+ "\"reason\":\"上游拒绝代理出口 IP,换 IP 重试仍未放行\",\"retryable\":false}"));
try {
BrandCheckClient client = newClient(server, 1);
BrandCheckClient.BrandCheckResponse response = client.check("b1", "Terms", 7L);
assertThat(response.getReason()).contains("代理出口");
assertThat(response.isRetryable()).isFalse();
} finally {
server.stop(0);
}
}
@Test
void egressRejectionStopsBatchAfterTwoBrands() throws Exception {
AtomicInteger requests = new AtomicInteger();
HttpServer server = startBrandCheckServer(exchange -> {
requests.incrementAndGet();
respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[\"x\"],"
+ "\"reason\":\"上游拒绝代理出口 IP,换 IP 重试仍未放行\",\"retryable\":false}");
});
try {
BrandCheckClient client = newClient(server, 10);
List<String> brands = List.of("b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8");
BrandCheckClient.BrandCheckBatchResult result = client.checkAll(brands, "Terms", 7L);
assertThat(requests.get())
// 检查池并发 3:三条在飞 + 一条在首个响应回来后替补进来,最多 4 次;
// 关键是这个上界与品牌数无关,不能回到「品牌数 × 10 轮」的放大形态
.as("出口被拒后只允许并发在飞的前几个品牌各打一次,不能按品牌数×10 轮放大")
.isBetween(1, 4);
assertThat(result.hasQueryFailedData()).isTrue();
assertThat(result.reason()).contains("代理出口");
} finally {
server.stop(0);
}
}
/**
* 短路判据必须保守:只有"跑满整条阶梯仍每轮查询失败"才成立。
* 成功品牌首轮就返回、从不置标志,所以同一批里的个别失败品牌要照旧拿满它的重试机会——
@@ -31,6 +31,24 @@ import static org.mockito.Mockito.when;
class PriceTrackLoopRunServiceTest {
@Test
void pausedAccountCannotDispatchNext() {
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
loopRunMapper, mock(FileTaskMapper.class), new ObjectMapper(),
mock(ZiniaoShopSwitchService.class));
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService guard =
mock(com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.class);
when(guard.isPaused(1L)).thenReturn(true);
org.springframework.test.util.ReflectionTestUtils.setField(service, "taskGuard", guard);
BusinessException error = assertThrows(BusinessException.class,
() -> service.dispatchNext(743L, 1L));
assertTrue(error.getMessage().contains("账号任务已暂停"));
verify(loopRunMapper, never()).selectById(743L);
}
@Test
void syncLoopRunAfterChildTerminalUsesChildContextWhenLoopRowAlreadyDrifted() throws Exception {
ObjectMapper objectMapper = new ObjectMapper();
@@ -0,0 +1,137 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.CostGuardProperties;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.notification.service.NotificationService;
import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper;
import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDate;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class CostGuardSchedulerTest {
@Mock
private UserSecretUsageMapper usageMapper;
@Mock
private UserTaskGuardService guardService;
@Mock
private SharedSecretMonitorService sharedSecretMonitorService;
@Mock
private NotificationDispatchService notificationDispatchService;
@Mock
private DistributedJobLockService distributedJobLockService;
private CostGuardScheduler scheduler;
@BeforeEach
void setUp() {
CostGuardProperties properties = new CostGuardProperties();
properties.setProxyWindowThreshold(10);
properties.setLlmWindowCallsThreshold(5);
properties.setLlmWindowTokensThreshold(1_000);
properties.setDailyProxyThreshold(100);
properties.setDailyLlmCallsThreshold(100);
properties.setDailyLlmTokensThreshold(10_000);
scheduler = new CostGuardScheduler(properties, usageMapper, guardService,
sharedSecretMonitorService, notificationDispatchService, distributedJobLockService);
}
@Test
void firstScanOnlyBuildsBaseline() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 50, 0)));
when(usageMapper.sumForUser(eq(7L), any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 50, 0)));
scheduler.runOnce();
verify(guardService, never()).pause(anyLong(), anyString());
assertThat(scheduler.todayTotals(7L).proxy).isEqualTo(50);
}
@Test
void proxyWindowJumpPausesAndNotifies() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
.thenReturn(List.of(row(7L, "proxy", 12, 0)));
when(guardService.isPaused(7L)).thenReturn(false);
scheduler.runOnce();
scheduler.runOnce();
verify(guardService).pause(eq(7L), contains("代理提取 12 次"));
verify(notificationDispatchService).pushToUser(eq(7L),
eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR),
anyString(), contains("继续任务"), isNull());
}
@Test
void llmCallsWindowJumpPauses() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(8L, "similar-asin", 0, 0)))
.thenReturn(List.of(row(8L, "similar-asin", 6, 100)));
when(guardService.isPaused(8L)).thenReturn(false);
scheduler.runOnce();
scheduler.runOnce();
verify(guardService).pause(eq(8L), contains("LLM 调用 6 次"));
}
@Test
void dailyCrossingOnlyPausesOnceAndRespectsResetBaseline() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(9L, "proxy", 99, 0)))
.thenReturn(List.of(row(9L, "proxy", 100, 0)))
.thenReturn(List.of(row(9L, "proxy", 101, 0)));
when(guardService.isPaused(9L)).thenReturn(false);
scheduler.runOnce();
scheduler.runOnce();
scheduler.resetBaseline(9L);
scheduler.runOnce();
verify(guardService).pause(eq(9L), contains("当日上限 100 次"));
}
@Test
void alreadyPausedUserIsNotPausedAgain() {
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
.thenReturn(List.of(row(7L, "proxy", 50, 0)));
when(guardService.isPaused(7L)).thenReturn(true);
scheduler.runOnce();
scheduler.runOnce();
verify(guardService, never()).pause(anyLong(), anyString());
}
private UserSecretUsageAggregate row(long userId, String module, long calls, long tokens) {
UserSecretUsageAggregate row = new UserSecretUsageAggregate();
row.setUserId(userId);
row.setModuleKey(module);
row.setCallCount(calls);
row.setTokens(tokens);
return row;
}
}
@@ -0,0 +1,101 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.config.CostGuardProperties;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.notification.service.NotificationService;
import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper;
import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity;
import com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class SharedSecretMonitorServiceTest {
@Mock
private UserApiSecretMapper userApiSecretMapper;
@Mock
private NotificationDispatchService notificationDispatchService;
private CostGuardProperties properties;
private SharedSecretMonitorService service;
@BeforeEach
void setUp() {
properties = new CostGuardProperties();
properties.setSharedAccountsThreshold(3);
service = new SharedSecretMonitorService(userApiSecretMapper, properties,
notificationDispatchService);
}
@Test
void threeAccountsSharingProxyProducesWarningAndDailyNotice() {
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
row(1L, "proxy", "same-cipher"),
row(2L, "proxy", "same-cipher"),
row(3L, "proxy", "same-cipher"),
row(4L, "proxy", "own-cipher")));
service.refresh();
List<SharedSecretWarningVo> warnings = service.warningsFor(1L);
assertThat(warnings).hasSize(1);
assertThat(warnings.get(0).getType()).isEqualTo("proxy");
assertThat(warnings.get(0).getAccounts()).isEqualTo(3);
assertThat(warnings.get(0).getMessage()).contains("3 个账号共用").contains("拆分独立密钥");
assertThat(service.warningsFor(4L)).isEmpty();
verify(notificationDispatchService, times(3)).pushToUser(anyLong(),
eq(NotificationService.SCENE_SECRET_SHARED), eq(NotificationService.LEVEL_WARNING),
contains("3 个账号共用"), contains("拆分独立密钥"), anyString());
}
@Test
void belowThresholdProducesNoWarning() {
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
row(1L, "similar-asin", "same"),
row(2L, "similar-asin", "same")));
service.refresh();
assertThat(service.warningsFor(1L)).isEmpty();
verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(),
anyString(), anyString(), anyString());
}
@Test
void llmModuleIsTypedAsLlm() {
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
row(1L, "appearance-patent", "llm-key"),
row(2L, "appearance-patent", "llm-key"),
row(3L, "appearance-patent", "llm-key")));
service.refresh();
assertThat(service.warningsFor(2L).get(0).getType()).isEqualTo("llm");
assertThat(service.warningsFor(2L).get(0).getModuleLabel()).isEqualTo("外观专利密钥");
}
private UserApiSecretEntity row(long userId, String moduleKey, String cipher) {
UserApiSecretEntity row = new UserApiSecretEntity();
row.setUserId(userId);
row.setModuleKey(moduleKey);
row.setSecretValue(cipher);
return row;
}
}
@@ -0,0 +1,66 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.modules.usersecret.mapper.UserTaskGuardMapper;
import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class UserTaskGuardServiceTest {
@Mock
private UserTaskGuardMapper guardMapper;
@Test
void pausedStateIsCachedForHotPaths() {
UserTaskGuardEntity row = new UserTaskGuardEntity();
row.setUserId(7L);
row.setPaused(true);
row.setReason("代理提取异常");
row.setPausedAt(LocalDateTime.now());
when(guardMapper.selectById(7L)).thenReturn(row);
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
assertThat(service.isPaused(7L)).isTrue();
assertThat(service.isPaused(7L)).isTrue();
verify(guardMapper, org.mockito.Mockito.times(1)).selectById(7L);
}
@Test
void pauseAndResumeClearCacheImmediately() {
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
when(guardMapper.selectById(7L)).thenReturn(null);
service.pause(7L, "LLM 消耗异常");
verify(guardMapper).pause(7L, "LLM 消耗异常");
service.resume(7L);
verify(guardMapper).resume(7L);
assertThat(service.isPaused(7L)).isFalse();
}
@Test
void missingUserIsNeverPaused() {
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
assertThat(service.isPaused(null)).isFalse();
assertThat(service.isPaused(0L)).isFalse();
}
@Test
void mapperFailureFallsBackToNotPaused() {
when(guardMapper.selectById(7L)).thenThrow(new IllegalStateException("db down"));
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
assertThat(service.isPaused(7L)).isFalse();
}
}