fix(通知): 麦象不可读不再折叠成「无异常」、探活 4xx 不冒充正常、交付不完整独立成 scene,前端入队拒绝按机器码决定重试

- MaixiangConsoleClient 新增 probeReadable/classifyReadout,把四种「读不出来」的形状与
  「确实无异常」分开;MaixiangAnomalyScanner 在六项扫描前先过这道闸,读不通就喊一条
  天级去重的管理员异常并退出本轮——此前后台挂掉时六项一致报「无异常」,服务探测还是绿的。
- NotificationScanScheduler 的探活改三态 ALIVE/UNVERIFIED/DOWN:401/403/404 说明请求被
  挡在业务之外,拿不到健康证据,不能再按 <500 判成「服务正常」。
- task_partial(交付不完整)从 task_failed 里拆出来独立成 scene,仍归任务异常大类;
  NotificationDispatchService 的管理员行改用 SCENE_TASK_PARTIAL。
- 前端 dispatch-guard 用机器码 shop_busy/queue_full 判定「等一会儿再推就会成」,
  暂缓 20s×15 封顶;此前页面靠 includes('当前店铺正在执行中') 匹配中文,而文案中间
  插了店铺名,永远匹配不到⇒同店互斥被写成任务失败。

护栏:后端 notification 六支用例(含 MaixiangConsoleClientTest 15 例、
MaixiangAnomalyScannerTest 24 例)、前端 dispatch-guard 与 enqueue-rejection-wiring 两支;
后端全量 3827 例 0 红、前端 717 例 0 失败;变异 M1-M7、M12-M14 共 10/10 杀。
This commit is contained in:
2026-10-06 11:46:54 +08:00
parent 4596e7d840
commit 7887e7ee37
18 changed files with 545 additions and 47 deletions
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.notification.client;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.NotificationProperties;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.ConsoleReadout;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
@@ -142,6 +143,28 @@ class MaixiangConsoleClientTest {
assertThat(client.parseEnvelope("not-json", "test")).isNull();
}
@Test
void classifyReadoutNamesEveryUnreadableShape() {
// 这四种形态原先都被折叠成「空数据」,扫描器看不出自己已经瞎了
assertThat(client.classifyReadout(null, "Connection refused: connect").readable()).isFalse();
assertThat(client.classifyReadout(null, "Connection refused: connect").reason())
.contains("Connection refused");
assertThat(client.classifyReadout("", null).readable()).isFalse();
assertThat(client.classifyReadout("", null).reason()).contains("空");
assertThat(client.classifyReadout("<h1>502 Bad Gateway</h1>", null).readable()).isFalse();
assertThat(client.classifyReadout("<h1>502 Bad Gateway</h1>", null).reason()).contains("JSON");
// 令牌失效 / 接口未部署 / 契约变更:HTTP 200 但 code!=1,mes 必须进告警文案
ConsoleReadout rejected = client.classifyReadout("{\"code\":-1,\"mes\":\"令牌无效\",\"data\":null}", null);
assertThat(rejected.readable()).isFalse();
assertThat(rejected.reason()).contains("令牌无效");
}
@Test
void classifyReadoutAcceptsSuccessEnvelope() {
assertThat(client.classifyReadout("{\"code\":1,\"mes\":\"成功\",\"data\":{\"queues\":[]}}", null).readable())
.isTrue();
}
@Test
void parseToleratesNullUidAndBadTime() {
BatchTaskPage page = client.parseBatchTaskPage(client.parseEnvelope(
@@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.ConsoleReadout;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
@@ -72,6 +73,48 @@ class MaixiangAnomalyScannerTest {
verify(consoleClient, never()).queueStatus();
}
// ======================== 后台可读性 ========================
@Test
void scanAlertsOnceAndSkipsChecksWhenConsoleUnreadable() {
// 六项检查全靠后台接口取数:读不到时它们只会「查无异常」,整轮扫描静默失效
when(consoleClient.configured()).thenReturn(true);
when(dispatch.prepareAdminAudience()).thenReturn(audience);
when(consoleClient.probeReadable())
.thenReturn(ConsoleReadout.unreadable("接口返回失败:token 无效"));
newScanner().scan();
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象后台数据不可读"),
content.capture(), key.capture(), isNull());
assertThat(content.getValue()).contains("token 无效");
// scan() 自己取当前时间(NOW 常量注入不进来),这里钉住「按自然日去重」的形状而不是某一天
assertThat(key.getValue()).matches("maixiang_console_unreadable:\\d{8}");
verify(consoleClient, never()).batchTasks(anyInt(), any(), anyInt());
verify(consoleClient, never()).singleTasks(anyInt(), any(), anyInt());
verify(consoleClient, never()).queueStatus();
verify(consoleClient, never()).workers();
}
@Test
void scanRunsChecksWhenConsoleReadable() {
when(consoleClient.configured()).thenReturn(true);
when(dispatch.prepareAdminAudience()).thenReturn(audience);
when(consoleClient.probeReadable()).thenReturn(ConsoleReadout.OK);
when(consoleClient.workers()).thenReturn(WorkerSnapshot.EMPTY);
when(consoleClient.queueStatus()).thenReturn(List.of());
when(consoleClient.batchTasks(anyInt(), any(), anyInt())).thenReturn(BatchTaskPage.EMPTY);
when(consoleClient.singleTasks(anyInt(), any(), anyInt())).thenReturn(SingleTaskPage.EMPTY);
newScanner().scan();
verify(consoleClient, times(1)).queueStatus();
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
// ======================== 批量任务停滞 ========================
@Test
@@ -373,6 +416,7 @@ class MaixiangAnomalyScannerTest {
properties.setMaixiangConsoleToken("token");
when(consoleClient.configured()).thenReturn(true);
when(dispatch.prepareAdminAudience()).thenReturn(audience);
when(consoleClient.probeReadable()).thenReturn(ConsoleReadout.OK);
// 仅批量任务停滞检查(status=0 拉取)抛异常
doThrow(new RuntimeException("麦象接口超时")).when(consoleClient).batchTasks(eq(0), any(), anyInt());
doReturn(BatchTaskPage.EMPTY).when(consoleClient).batchTasks(eq(1), any(), anyInt());
@@ -40,7 +40,7 @@ class NotificationDispatchServiceRecheckTest {
void pushRecheckOutcomeNotifiesUserWithInfoLevelWhenAllResolved() {
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_partial"),
eq("info"), contains("补测完成"), contains("新增结果 3 行"), startsWith("collect_recheck:29842:"));
}
@@ -48,7 +48,7 @@ class NotificationDispatchServiceRecheckTest {
void pushRecheckOutcomeUsesWarningWhileRowsStillPending() {
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 0, 5, 0);
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_partial"),
eq("warning"), contains("补测完成"), contains("仍有 5 行"), anyString());
}
@@ -66,7 +66,7 @@ class NotificationDispatchServiceRecheckTest {
void pushRecheckOutcomeAdvisesManualRerunWhenAllExhausted() {
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 0, 4);
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_partial"),
eq("warning"), contains("未能完成"), contains("共 4 行"), startsWith("collect_recheck:29842:"));
}
@@ -12,6 +12,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -159,6 +160,29 @@ class NotificationDispatchServiceTest {
assertThat(pushed).isEqualTo(3);
}
/**
* 「部分结果缺失」必须落在自己的 scene:它和终态失败共用 task_failed 时,巡检按
* task_failed 独立桶数报「窗口内 N 条独立失败」会把「任务成功、少跑几行」也算成任务失败
* (2026-10-05 值班:窗口 22 行里 20 行是这种)。去重键与大类都不变,只拆 scene。
*/
@Test
void pushPartialResultUsesTaskPartialScene() {
AdminUserEntity superAdmin = admin(1L, "超管甲");
when(adminUserMapper.selectList(any())).thenReturn(List.of(superAdmin));
when(adminAuthSupport.currentRole(superAdmin)).thenReturn("super_admin");
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
.thenReturn(true);
service.pushPartialResult(20L, 3301L, "BRAND", "BRAND-3301", "12 个品牌查询失败");
verify(notificationService).pushOrRefresh(eq(20L), eq(NotificationService.AUDIENCE_USER),
eq(NotificationService.SCENE_TASK_PARTIAL), eq(NotificationService.LEVEL_WARNING),
contains("任务部分结果缺失"), contains("12 个品牌查询失败"), eq("task_partial:3301"));
verify(notificationService).pushOrRefresh(eq(1L), eq(NotificationService.AUDIENCE_ADMIN),
eq(NotificationService.SCENE_TASK_PARTIAL), eq(NotificationService.LEVEL_WARNING),
contains("用户"), anyString(), eq("task_partial_admin:3301:1"));
}
@Test
void pushToUserUsesUserAudience() {
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
@@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -184,6 +185,100 @@ class NotificationScanSchedulerTest {
assertThat(keyBase.getValue()).matches("service_down:brand-service:\\d{8}");
}
/** 状态码三态口径:4xx 里只有 401/403/404 是「探活不可信」,其余按故障算。 */
@Test
void httpStatusClassifiesIntoThreeProbeVerdicts() {
assertThat(NotificationScanScheduler.probeOutcomeFor(200, 5).verdict())
.isEqualTo(NotificationScanScheduler.ProbeVerdict.ALIVE);
assertThat(NotificationScanScheduler.probeOutcomeFor(302, 5).verdict())
.isEqualTo(NotificationScanScheduler.ProbeVerdict.ALIVE);
for (int status : new int[]{401, 403, 404}) {
NotificationScanScheduler.ProbeOutcome outcome =
NotificationScanScheduler.probeOutcomeFor(status, 5);
assertThat(outcome.verdict())
.as("HTTP %d 不能再被当成「服务存活」:路径没人接/鉴权失败都证不了业务可用", status)
.isEqualTo(NotificationScanScheduler.ProbeVerdict.UNVERIFIED);
assertThat(outcome.detail()).contains("HTTP " + status).contains("未验证");
assertThat(outcome.ok()).isFalse();
}
for (int status : new int[]{400, 405, 429, 500, 502, 503}) {
assertThat(NotificationScanScheduler.probeOutcomeFor(status, 5).verdict())
.as("HTTP %d 是故障", status)
.isEqualTo(NotificationScanScheduler.ProbeVerdict.DOWN);
}
}
/** 真发一次 HTTP:404 判「探活不可信」,并且不重试(确定性结果打两次只多一份日志)。 */
@Test
void probeAgainstHttp404IsUnverifiedAndNotRetried() throws Exception {
java.util.concurrent.atomic.AtomicInteger hits = new java.util.concurrent.atomic.AtomicInteger();
com.sun.net.httpserver.HttpServer server = startFixedStatusServer(404, hits);
NotificationScanScheduler.ProbeOutcome outcome;
try {
outcome = newScheduler().probeHttpWithRetry(
"http://127.0.0.1:" + server.getAddress().getPort() + "/api/version");
} finally {
server.stop(0);
}
assertThat(outcome.verdict()).isEqualTo(NotificationScanScheduler.ProbeVerdict.UNVERIFIED);
assertThat(outcome.detail()).contains("未验证");
assertThat(hits.get()).isEqualTo(1);
}
/** 反向对照:真故障仍然「失败立即重试一次,两次都失败才算故障」,别把瞬时抖动报出去。 */
@Test
void probeOnHttp503RetriesOnceThenDeclaresOutage() throws Exception {
java.util.concurrent.atomic.AtomicInteger hits = new java.util.concurrent.atomic.AtomicInteger();
com.sun.net.httpserver.HttpServer server = startFixedStatusServer(503, hits);
NotificationScanScheduler.ProbeOutcome outcome;
try {
outcome = newScheduler().probeHttpWithRetry(
"http://127.0.0.1:" + server.getAddress().getPort() + "/api/version");
} finally {
server.stop(0);
}
assertThat(outcome.verdict()).isEqualTo(NotificationScanScheduler.ProbeVerdict.DOWN);
assertThat(hits.get()).isEqualTo(2);
}
/** 404 不推「服务不可用」通知——三态改造不许顺手把它升级成误报(现网最大一类误报源)。 */
@Test
void probeOnHttp404DoesNotPushServiceDown() throws Exception {
com.sun.net.httpserver.HttpServer server = startFixedStatusServer(404,
new java.util.concurrent.atomic.AtomicInteger());
try {
properties.setBrandServiceUrl("http://127.0.0.1:" + server.getAddress().getPort());
properties.setPriceTrackApiUrl("");
properties.setJikipProbeEnabled(false);
newScheduler().probeServices();
} finally {
server.stop(0);
}
verify(dispatch, never()).pushToAdmins(anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
private com.sun.net.httpserver.HttpServer startFixedStatusServer(
int status, java.util.concurrent.atomic.AtomicInteger hits) throws java.io.IOException {
com.sun.net.httpserver.HttpServer server =
com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/", exchange -> {
hits.incrementAndGet();
byte[] body = "probe".getBytes(java.nio.charset.StandardCharsets.UTF_8);
exchange.sendResponseHeaders(status, body.length);
exchange.getResponseBody().write(body);
exchange.close();
});
server.setExecutor(null);
server.start();
return server;
}
private void stubAdminAudience() {
when(dispatch.prepareAdminAudience()).thenReturn(
new NotificationDispatchService.AdminAudience(List.of(), Set.of(), Map.of()));
@@ -260,6 +260,22 @@ class NotificationServiceTest {
.isEqualTo(NotificationService.CATEGORY_SYSTEM_NOTICE);
}
/**
* task_partial(交付不完整)与 task_failed(终态失败)是两个 scene,但用户可见大类都必须是
* 「任务异常」——拆开 scene 是为了让巡检/看板不再把「少跑了几行」数成「任务失败」,
* 不是为了改铃铛的分类。漏登记会兜底进「系统通知」,用户按「任务异常」筛就凭空少一批。
*/
@Test
void taskPartialAndTaskFailedAreSeparateScenesInSameCategory() {
assertThat(NotificationService.SCENE_TASK_PARTIAL).isNotEqualTo(NotificationService.SCENE_TASK_FAILED);
assertThat(NotificationService.categoryOf(NotificationService.SCENE_TASK_PARTIAL))
.isEqualTo(NotificationService.CATEGORY_TASK_ERROR);
assertThat(NotificationService.categoryOf(NotificationService.SCENE_TASK_FAILED))
.isEqualTo(NotificationService.CATEGORY_TASK_ERROR);
assertThat(NotificationService.scenesOfCategory(NotificationService.CATEGORY_TASK_ERROR))
.contains(NotificationService.SCENE_TASK_FAILED, NotificationService.SCENE_TASK_PARTIAL);
}
/** 类型筛选落到 SQL 上是 scene IN (...);空/未知分类不追加条件,避免前端传错值就查不到数据。 */
@Test
@SuppressWarnings("unchecked")