feat(通知): 新增「采集机掉线」告警,并把队列积压判据改成「有活没人干」
背景(2026-09-29):麦象采集机 15:19 起整机停摆 6 小时(5 个 worker 全没,机器
无开机自启),现有告警完全没发现 —— 五类检查全靠「任务 update_time 不新鲜」间接
推断,一次停摆被摊成几十条"任务失败";队列积压又卡在 pending≥300 的阈值上
(当时停在 210),用户看到的是大面积业务失败,误导排查方向。
- 新增第 6 项检查「采集机掉线」:读 18960 新接口 /api/console/workers 的
machineIdleSeconds(采集端打点,见 maixiang_run_task 同批提交),超阈值(默认 10
分钟)推一条管理员告警,文案含逐信号静默时长便于定位。
* 按**机器级**判活而不是逐 worker:变体采集领到长任务后不再调 /batch/task/get,
逐 worker 判会在长任务正常运行时误报「vc 掉线」(实测踩到)。
* machineIdleSeconds=null(从未上报)视为未知并跳过,避免刚上线误报一次。
- 队列积压判据改为 pending≥阈值 **且 processing=0**,阈值默认 300→100:
只看 pending 两边都不讨好(低了误报"一时来不及",高了真出事也不报)。
单 worker 死掉(机器还活着)由这条覆盖。
- 配置:新增 AIIMAGE_NOTIFICATION_MAIXIANG_SILENT_MINUTES(默认 10)。
This commit is contained in:
+34
@@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
@@ -86,6 +87,39 @@ class MaixiangConsoleClientTest {
|
||||
assertThat(queues.get(0).processing()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseWorkersReadsMachineIdleAndPerSignalIdle() {
|
||||
WorkerSnapshot empty = client.parseWorkers(
|
||||
client.parseEnvelope("{\"code\":1,\"mes\":\"成功\",\"data\":{}}", "test"));
|
||||
assertThat(empty.machineIdleSeconds()).isNull();
|
||||
assertThat(empty.workers()).isEmpty();
|
||||
|
||||
WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
|
||||
"{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":12,\"machineLastSeenAt\":1790000000,"
|
||||
+ "\"workers\":["
|
||||
+ "{\"signal\":\"pt\",\"label\":\"跟价\",\"lastSeenAt\":1790000000,\"idleSeconds\":12},"
|
||||
+ "{\"signal\":\"vc\",\"label\":\"变体采集\",\"lastSeenAt\":null,\"idleSeconds\":null}]}}",
|
||||
"test"));
|
||||
|
||||
assertThat(snapshot.machineIdleSeconds()).isEqualTo(12L);
|
||||
assertThat(snapshot.workers()).hasSize(2);
|
||||
assertThat(snapshot.workers().get(0).signal()).isEqualTo("pt");
|
||||
assertThat(snapshot.workers().get(0).label()).isEqualTo("跟价");
|
||||
assertThat(snapshot.workers().get(0).idleSeconds()).isEqualTo(12L);
|
||||
// JSON null 必须解析成 null 而不是 0:0 会被当成「刚刚还在打点」,把掉线判成正常
|
||||
assertThat(snapshot.workers().get(1).idleSeconds()).isNull();
|
||||
assertThat(snapshot.workers().get(1).lastSeenEpoch()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseWorkersTreatsNullMachineIdleAsUnknown() {
|
||||
// 机器级静默为 null = 从未上报(≠ 0 秒)——调用方据此跳过掉线判断,避免刚上线误报
|
||||
WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
|
||||
"{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":null,\"workers\":[]}}", "test"));
|
||||
|
||||
assertThat(snapshot.machineIdleSeconds()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseBatchResultTotalDistinguishesUnsupportedTable() {
|
||||
// 已注册结果表:返回真实行数
|
||||
|
||||
+119
-3
@@ -8,6 +8,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
@@ -225,8 +227,9 @@ class MaixiangAnomalyScannerTest {
|
||||
// ======================== 队列积压 ========================
|
||||
|
||||
@Test
|
||||
void queueBacklogPushesWhenOverThreshold() {
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 500, 3)));
|
||||
void queueBacklogPushesWhenPendingPiledUpAndNobodyConsuming() {
|
||||
// 有活没人干:210 条堆积 + processing=0(2026-09-29 停摆时的真实形态,当时 300 阈值没触发)
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 210, 0)));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
@@ -236,7 +239,31 @@ class MaixiangAnomalyScannerTest {
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"),
|
||||
content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull());
|
||||
assertThat(content.getValue()).contains("L1 待处理 500 / 处理中 3");
|
||||
assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queueBacklogIgnoresPendingWhenConsumerIsWorking() {
|
||||
// 回归守卫:501 条堆积但处理中有 3 个 → 采集端在正常消费,不该报异常
|
||||
// (旧判据只看 pending 阈值,会把这种情况误报成故障)
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 501, 3)));
|
||||
|
||||
newScanner().checkQueueBacklog(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queueBacklogPushesWhenProcessingPiledUp() {
|
||||
// processing 超阈值 = 领走了却不 ACK,仍有独立意义(与 pending 分支互不干扰)
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 0, 150)));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScanner().checkQueueBacklog(audience, NOW);
|
||||
|
||||
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(),
|
||||
eq("maixiang_queue_backlog:2026091512"), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -248,6 +275,95 @@ class MaixiangAnomalyScannerTest {
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
// ======================== 采集机掉线(机器级判活) ========================
|
||||
|
||||
@Test
|
||||
void collectorSilentPushesWhenMachineStaysSilent() {
|
||||
// 整机停摆:机器级静默 11 分钟 → 一条「采集机掉线」,而不是几十条任务失败
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(700L, List.of(
|
||||
new WorkerItem("pt", "跟价", 1_790_000_000L, 700L),
|
||||
new WorkerItem("vc", "变体采集", 1_790_000_040L, 660L),
|
||||
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 700L))));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:采集机掉线"),
|
||||
content.capture(), eq("maixiang_silent:2026091512"), isNull());
|
||||
assertThat(content.getValue())
|
||||
.contains("静默 11 分钟")
|
||||
.contains("跟价 静默 11 分钟")
|
||||
.contains("cookie 通道")
|
||||
.contains("无开机自启");
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentIgnoresStaleSignalWhileMachineAlive() {
|
||||
// 核心回归(2026-09-29 实测发现的坑):变体采集领到长任务后不再调 /batch/task/get,
|
||||
// 于是 vc 信号陈旧,但机器明显活着(ck 0 秒前刚打点)→ 机器级静默 0,绝不能报掉线。
|
||||
// 服务端把「所有信号里最新的一个」当机器级判活值就是为了这个场景。
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(0L, List.of(
|
||||
new WorkerItem("pt", "跟价", 1_790_000_000L, 5L),
|
||||
new WorkerItem("vc", "变体采集", 1_780_000_000L, 36_000L),
|
||||
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 0L))));
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentIgnoresFreshMachine() {
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(599L, List.of(
|
||||
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 599L))));
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentSkipsWhenNeverReported() {
|
||||
// machineIdleSeconds=null = 从未上报(刚上线/清过 Redis/接口刚部署)→ 跳过而不是误报掉线;
|
||||
// 这类情况由「队列积压 / 任务滞留」覆盖
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(null, List.of(
|
||||
new WorkerItem("pt", "跟价", null, null))));
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentShowsNeverReportedSignalInContent() {
|
||||
// 告警文案要能区分「某信号从未上报」与「静默 N 分钟」,否则排查时看不出是哪一路缺
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(900L, List.of(
|
||||
new WorkerItem("pt", "跟价", 1_790_000_000L, 900L),
|
||||
new WorkerItem("vc", "变体采集", null, null))));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(),
|
||||
content.capture(), anyString(), any());
|
||||
assertThat(content.getValue()).contains("跟价 静默 15 分钟").contains("变体采集 (从未上报)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentSkipsWhenEndpointUnavailable() {
|
||||
// 未部署 /api/console/workers(空快照):跳过而不是误报掉线
|
||||
when(consoleClient.workers()).thenReturn(WorkerSnapshot.EMPTY);
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
// ======================== 容错 ========================
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user