feat(cost-guard): 代理/LLM 损耗监控与账号级任务暂停
- 每分钟扫描用量增量/日累计,异常置暂停闸并推站内通知 - 品牌补测、跟价派发、外观激活、品牌检测与两个 LLM 客户端统一接闸 - 同一代理/LLM 密钥被 3 个以上账号共用时给出用户提示 - 品牌检测出口拒绝批级熔断;LLM 同 key 共享并发/间隔/429 惩罚 - V170 新增 biz_user_task_guard
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
package com.nanri.aiimage.common.retry;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.client.HttpClientErrorException;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class LlmKeyGateTest {
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
LlmKeyGate.resetForTests();
|
||||
}
|
||||
|
||||
@Test
|
||||
void recognizesRateLimitThroughExceptionChain() {
|
||||
assertThat(LlmKeyGate.isRateLimited(
|
||||
new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS))).isTrue();
|
||||
assertThat(LlmKeyGate.isRateLimited(
|
||||
new IllegalStateException("wrapped",
|
||||
new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)))).isTrue();
|
||||
assertThat(LlmKeyGate.isRateLimited(
|
||||
new IllegalStateException("read timeout"))).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void permitIsReleasedAfterUse() {
|
||||
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire("shared-key")) {
|
||||
assertThat(ignored).isNotNull();
|
||||
}
|
||||
// 能再次拿到同一个 key 的许可,说明 close 归还了信号量
|
||||
try (LlmKeyGate.Permit ignored = LlmKeyGate.acquire("shared-key")) {
|
||||
assertThat(ignored).isNotNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
+79
@@ -81,6 +81,31 @@ class BrandCheckClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void pausedAccountStopsBatchWithoutHittingUpstream() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
HttpServer server = startBrandCheckServer(exchange -> {
|
||||
attempts.incrementAndGet();
|
||||
respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[]}");
|
||||
});
|
||||
try {
|
||||
BrandCheckClient client = newClient(server, 3);
|
||||
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService guard =
|
||||
mock(com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.class);
|
||||
when(guard.isPaused(7L)).thenReturn(true);
|
||||
ReflectionTestUtils.setField(client, "taskGuard", guard);
|
||||
|
||||
BrandCheckClient.BrandCheckBatchResult result =
|
||||
client.checkAll(List.of("Apple", "Nike"), "Terms", 7L);
|
||||
|
||||
assertThat(attempts.get()).as("账号已暂停时一个上游请求都不该发").isZero();
|
||||
assertThat(result.queryFaildData()).containsExactly("Apple", "Nike");
|
||||
assertThat(result.reason()).contains("账号任务已暂停");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryFailureRetriesUntilSuccess() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
@@ -176,6 +201,60 @@ class BrandCheckClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 出口被上游连续拒绝时,整批不再给每个品牌各发一次首轮探针。
|
||||
*
|
||||
* <p>2026-10-23 账单形状:16890 每个品牌最多换 2 个出口 IP,Java 再按品牌重试
|
||||
* 10 轮;一批几千个品牌就是几千×2 次提取。出口熔断证据只需前 2 个品牌,
|
||||
* 后面的品牌直接判查询失败转补测。
|
||||
*/
|
||||
@Test
|
||||
void egressClassifierMatchesUpstreamReason() {
|
||||
assertThat(com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice.isEgressRejected(
|
||||
"上游拒绝代理出口 IP,换 IP 重试仍未放行")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void egressReasonParsesFromUpstream() throws Exception {
|
||||
HttpServer server = startBrandCheckServer(exchange -> respond(exchange,
|
||||
"{\"faild_data\":[],\"query_faild_data\":[\"x\"],"
|
||||
+ "\"reason\":\"上游拒绝代理出口 IP,换 IP 重试仍未放行\",\"retryable\":false}"));
|
||||
try {
|
||||
BrandCheckClient client = newClient(server, 1);
|
||||
BrandCheckClient.BrandCheckResponse response = client.check("b1", "Terms", 7L);
|
||||
assertThat(response.getReason()).contains("代理出口");
|
||||
assertThat(response.isRetryable()).isFalse();
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void egressRejectionStopsBatchAfterTwoBrands() throws Exception {
|
||||
AtomicInteger requests = new AtomicInteger();
|
||||
HttpServer server = startBrandCheckServer(exchange -> {
|
||||
requests.incrementAndGet();
|
||||
respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[\"x\"],"
|
||||
+ "\"reason\":\"上游拒绝代理出口 IP,换 IP 重试仍未放行\",\"retryable\":false}");
|
||||
});
|
||||
try {
|
||||
BrandCheckClient client = newClient(server, 10);
|
||||
List<String> brands = List.of("b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8");
|
||||
|
||||
BrandCheckClient.BrandCheckBatchResult result = client.checkAll(brands, "Terms", 7L);
|
||||
|
||||
assertThat(requests.get())
|
||||
// 检查池并发 3:三条在飞 + 一条在首个响应回来后替补进来,最多 4 次;
|
||||
// 关键是这个上界与品牌数无关,不能回到「品牌数 × 10 轮」的放大形态
|
||||
.as("出口被拒后只允许并发在飞的前几个品牌各打一次,不能按品牌数×10 轮放大")
|
||||
.isBetween(1, 4);
|
||||
assertThat(result.hasQueryFailedData()).isTrue();
|
||||
assertThat(result.reason()).contains("代理出口");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 短路判据必须保守:只有"跑满整条阶梯仍每轮查询失败"才成立。
|
||||
* 成功品牌首轮就返回、从不置标志,所以同一批里的个别失败品牌要照旧拿满它的重试机会——
|
||||
|
||||
+18
@@ -31,6 +31,24 @@ import static org.mockito.Mockito.when;
|
||||
|
||||
class PriceTrackLoopRunServiceTest {
|
||||
|
||||
@Test
|
||||
void pausedAccountCannotDispatchNext() {
|
||||
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||
loopRunMapper, mock(FileTaskMapper.class), new ObjectMapper(),
|
||||
mock(ZiniaoShopSwitchService.class));
|
||||
com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService guard =
|
||||
mock(com.nanri.aiimage.modules.usersecret.service.UserTaskGuardService.class);
|
||||
when(guard.isPaused(1L)).thenReturn(true);
|
||||
org.springframework.test.util.ReflectionTestUtils.setField(service, "taskGuard", guard);
|
||||
|
||||
BusinessException error = assertThrows(BusinessException.class,
|
||||
() -> service.dispatchNext(743L, 1L));
|
||||
|
||||
assertTrue(error.getMessage().contains("账号任务已暂停"));
|
||||
verify(loopRunMapper, never()).selectById(743L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void syncLoopRunAfterChildTerminalUsesChildContextWhenLoopRowAlreadyDrifted() throws Exception {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.CostGuardProperties;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationService;
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.dto.UserSecretUsageAggregate;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CostGuardSchedulerTest {
|
||||
|
||||
@Mock
|
||||
private UserSecretUsageMapper usageMapper;
|
||||
@Mock
|
||||
private UserTaskGuardService guardService;
|
||||
@Mock
|
||||
private SharedSecretMonitorService sharedSecretMonitorService;
|
||||
@Mock
|
||||
private NotificationDispatchService notificationDispatchService;
|
||||
@Mock
|
||||
private DistributedJobLockService distributedJobLockService;
|
||||
|
||||
private CostGuardScheduler scheduler;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
CostGuardProperties properties = new CostGuardProperties();
|
||||
properties.setProxyWindowThreshold(10);
|
||||
properties.setLlmWindowCallsThreshold(5);
|
||||
properties.setLlmWindowTokensThreshold(1_000);
|
||||
properties.setDailyProxyThreshold(100);
|
||||
properties.setDailyLlmCallsThreshold(100);
|
||||
properties.setDailyLlmTokensThreshold(10_000);
|
||||
scheduler = new CostGuardScheduler(properties, usageMapper, guardService,
|
||||
sharedSecretMonitorService, notificationDispatchService, distributedJobLockService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstScanOnlyBuildsBaseline() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 50, 0)));
|
||||
when(usageMapper.sumForUser(eq(7L), any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 50, 0)));
|
||||
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService, never()).pause(anyLong(), anyString());
|
||||
assertThat(scheduler.todayTotals(7L).proxy).isEqualTo(50);
|
||||
}
|
||||
|
||||
@Test
|
||||
void proxyWindowJumpPausesAndNotifies() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 12, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(false);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService).pause(eq(7L), contains("代理提取 12 次"));
|
||||
verify(notificationDispatchService).pushToUser(eq(7L),
|
||||
eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR),
|
||||
anyString(), contains("继续任务"), isNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void llmCallsWindowJumpPauses() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(8L, "similar-asin", 0, 0)))
|
||||
.thenReturn(List.of(row(8L, "similar-asin", 6, 100)));
|
||||
when(guardService.isPaused(8L)).thenReturn(false);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService).pause(eq(8L), contains("LLM 调用 6 次"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyCrossingOnlyPausesOnceAndRespectsResetBaseline() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 99, 0)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 100, 0)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 101, 0)));
|
||||
when(guardService.isPaused(9L)).thenReturn(false);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
scheduler.resetBaseline(9L);
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService).pause(eq(9L), contains("当日上限 100 次"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void alreadyPausedUserIsNotPausedAgain() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 50, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(true);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService, never()).pause(anyLong(), anyString());
|
||||
}
|
||||
|
||||
private UserSecretUsageAggregate row(long userId, String module, long calls, long tokens) {
|
||||
UserSecretUsageAggregate row = new UserSecretUsageAggregate();
|
||||
row.setUserId(userId);
|
||||
row.setModuleKey(module);
|
||||
row.setCallCount(calls);
|
||||
row.setTokens(tokens);
|
||||
return row;
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.config.CostGuardProperties;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationService;
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity;
|
||||
import com.nanri.aiimage.modules.usersecret.model.vo.SharedSecretWarningVo;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SharedSecretMonitorServiceTest {
|
||||
|
||||
@Mock
|
||||
private UserApiSecretMapper userApiSecretMapper;
|
||||
@Mock
|
||||
private NotificationDispatchService notificationDispatchService;
|
||||
|
||||
private CostGuardProperties properties;
|
||||
private SharedSecretMonitorService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
properties = new CostGuardProperties();
|
||||
properties.setSharedAccountsThreshold(3);
|
||||
service = new SharedSecretMonitorService(userApiSecretMapper, properties,
|
||||
notificationDispatchService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void threeAccountsSharingProxyProducesWarningAndDailyNotice() {
|
||||
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
|
||||
row(1L, "proxy", "same-cipher"),
|
||||
row(2L, "proxy", "same-cipher"),
|
||||
row(3L, "proxy", "same-cipher"),
|
||||
row(4L, "proxy", "own-cipher")));
|
||||
|
||||
service.refresh();
|
||||
|
||||
List<SharedSecretWarningVo> warnings = service.warningsFor(1L);
|
||||
assertThat(warnings).hasSize(1);
|
||||
assertThat(warnings.get(0).getType()).isEqualTo("proxy");
|
||||
assertThat(warnings.get(0).getAccounts()).isEqualTo(3);
|
||||
assertThat(warnings.get(0).getMessage()).contains("3 个账号共用").contains("拆分独立密钥");
|
||||
assertThat(service.warningsFor(4L)).isEmpty();
|
||||
verify(notificationDispatchService, times(3)).pushToUser(anyLong(),
|
||||
eq(NotificationService.SCENE_SECRET_SHARED), eq(NotificationService.LEVEL_WARNING),
|
||||
contains("3 个账号共用"), contains("拆分独立密钥"), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void belowThresholdProducesNoWarning() {
|
||||
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
|
||||
row(1L, "similar-asin", "same"),
|
||||
row(2L, "similar-asin", "same")));
|
||||
|
||||
service.refresh();
|
||||
|
||||
assertThat(service.warningsFor(1L)).isEmpty();
|
||||
verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(),
|
||||
anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void llmModuleIsTypedAsLlm() {
|
||||
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
|
||||
row(1L, "appearance-patent", "llm-key"),
|
||||
row(2L, "appearance-patent", "llm-key"),
|
||||
row(3L, "appearance-patent", "llm-key")));
|
||||
|
||||
service.refresh();
|
||||
|
||||
assertThat(service.warningsFor(2L).get(0).getType()).isEqualTo("llm");
|
||||
assertThat(service.warningsFor(2L).get(0).getModuleLabel()).isEqualTo("外观专利密钥");
|
||||
}
|
||||
|
||||
private UserApiSecretEntity row(long userId, String moduleKey, String cipher) {
|
||||
UserApiSecretEntity row = new UserApiSecretEntity();
|
||||
row.setUserId(userId);
|
||||
row.setModuleKey(moduleKey);
|
||||
row.setSecretValue(cipher);
|
||||
return row;
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserTaskGuardMapper;
|
||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserTaskGuardEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class UserTaskGuardServiceTest {
|
||||
|
||||
@Mock
|
||||
private UserTaskGuardMapper guardMapper;
|
||||
|
||||
@Test
|
||||
void pausedStateIsCachedForHotPaths() {
|
||||
UserTaskGuardEntity row = new UserTaskGuardEntity();
|
||||
row.setUserId(7L);
|
||||
row.setPaused(true);
|
||||
row.setReason("代理提取异常");
|
||||
row.setPausedAt(LocalDateTime.now());
|
||||
when(guardMapper.selectById(7L)).thenReturn(row);
|
||||
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
|
||||
|
||||
assertThat(service.isPaused(7L)).isTrue();
|
||||
assertThat(service.isPaused(7L)).isTrue();
|
||||
|
||||
verify(guardMapper, org.mockito.Mockito.times(1)).selectById(7L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseAndResumeClearCacheImmediately() {
|
||||
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
|
||||
when(guardMapper.selectById(7L)).thenReturn(null);
|
||||
|
||||
service.pause(7L, "LLM 消耗异常");
|
||||
verify(guardMapper).pause(7L, "LLM 消耗异常");
|
||||
|
||||
service.resume(7L);
|
||||
verify(guardMapper).resume(7L);
|
||||
assertThat(service.isPaused(7L)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingUserIsNeverPaused() {
|
||||
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
|
||||
|
||||
assertThat(service.isPaused(null)).isFalse();
|
||||
assertThat(service.isPaused(0L)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void mapperFailureFallsBackToNotPaused() {
|
||||
when(guardMapper.selectById(7L)).thenThrow(new IllegalStateException("db down"));
|
||||
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
|
||||
|
||||
assertThat(service.isPaused(7L)).isFalse();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user