feat(通知): 新增「采集机掉线」告警,并把队列积压判据改成「有活没人干」

背景(2026-09-29):麦象采集机 15:19 起整机停摆 6 小时(5 个 worker 全没,机器
无开机自启),现有告警完全没发现 —— 五类检查全靠「任务 update_time 不新鲜」间接
推断,一次停摆被摊成几十条"任务失败";队列积压又卡在 pending≥300 的阈值上
(当时停在 210),用户看到的是大面积业务失败,误导排查方向。

- 新增第 6 项检查「采集机掉线」:读 18960 新接口 /api/console/workers 的
  machineIdleSeconds(采集端打点,见 maixiang_run_task 同批提交),超阈值(默认 10
  分钟)推一条管理员告警,文案含逐信号静默时长便于定位。
  * 按**机器级**判活而不是逐 worker:变体采集领到长任务后不再调 /batch/task/get,
    逐 worker 判会在长任务正常运行时误报「vc 掉线」(实测踩到)。
  * machineIdleSeconds=null(从未上报)视为未知并跳过,避免刚上线误报一次。
- 队列积压判据改为 pending≥阈值 **且 processing=0**,阈值默认 300→100:
  只看 pending 两边都不讨好(低了误报"一时来不及",高了真出事也不报)。
  单 worker 死掉(机器还活着)由这条覆盖。
- 配置:新增 AIIMAGE_NOTIFICATION_MAIXIANG_SILENT_MINUTES(默认 10)。
This commit is contained in:
2026-09-29 21:45:16 +08:00
parent 209ec386a2
commit 76954280b1
6 changed files with 289 additions and 14 deletions
@@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
@@ -86,6 +87,39 @@ class MaixiangConsoleClientTest {
assertThat(queues.get(0).processing()).isEqualTo(3);
}
@Test
void parseWorkersReadsMachineIdleAndPerSignalIdle() {
WorkerSnapshot empty = client.parseWorkers(
client.parseEnvelope("{\"code\":1,\"mes\":\"成功\",\"data\":{}}", "test"));
assertThat(empty.machineIdleSeconds()).isNull();
assertThat(empty.workers()).isEmpty();
WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
"{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":12,\"machineLastSeenAt\":1790000000,"
+ "\"workers\":["
+ "{\"signal\":\"pt\",\"label\":\"跟价\",\"lastSeenAt\":1790000000,\"idleSeconds\":12},"
+ "{\"signal\":\"vc\",\"label\":\"变体采集\",\"lastSeenAt\":null,\"idleSeconds\":null}]}}",
"test"));
assertThat(snapshot.machineIdleSeconds()).isEqualTo(12L);
assertThat(snapshot.workers()).hasSize(2);
assertThat(snapshot.workers().get(0).signal()).isEqualTo("pt");
assertThat(snapshot.workers().get(0).label()).isEqualTo("跟价");
assertThat(snapshot.workers().get(0).idleSeconds()).isEqualTo(12L);
// JSON null 必须解析成 null 而不是 0:0 会被当成「刚刚还在打点」,把掉线判成正常
assertThat(snapshot.workers().get(1).idleSeconds()).isNull();
assertThat(snapshot.workers().get(1).lastSeenEpoch()).isNull();
}
@Test
void parseWorkersTreatsNullMachineIdleAsUnknown() {
// 机器级静默为 null = 从未上报(≠ 0 秒)——调用方据此跳过掉线判断,避免刚上线误报
WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
"{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":null,\"workers\":[]}}", "test"));
assertThat(snapshot.machineIdleSeconds()).isNull();
}
@Test
void parseBatchResultTotalDistinguishesUnsupportedTable() {
// 已注册结果表:返回真实行数
@@ -8,6 +8,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -225,8 +227,9 @@ class MaixiangAnomalyScannerTest {
// ======================== 队列积压 ========================
@Test
void queueBacklogPushesWhenOverThreshold() {
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 500, 3)));
void queueBacklogPushesWhenPendingPiledUpAndNobodyConsuming() {
// 有活没人干:210 条堆积 + processing=0(2026-09-29 停摆时的真实形态,当时 300 阈值没触发)
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 210, 0)));
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
.thenReturn(1);
@@ -236,7 +239,31 @@ class MaixiangAnomalyScannerTest {
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"),
content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull());
assertThat(content.getValue()).contains("L1 待处理 500 / 处理中 3");
assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费");
}
@Test
void queueBacklogIgnoresPendingWhenConsumerIsWorking() {
// 回归守卫:501 条堆积但处理中有 3 个 → 采集端在正常消费,不该报异常
// (旧判据只看 pending 阈值,会把这种情况误报成故障)
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 501, 3)));
newScanner().checkQueueBacklog(audience, NOW);
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
@Test
void queueBacklogPushesWhenProcessingPiledUp() {
// processing 超阈值 = 领走了却不 ACK,仍有独立意义(与 pending 分支互不干扰)
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 0, 150)));
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
.thenReturn(1);
newScanner().checkQueueBacklog(audience, NOW);
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(),
eq("maixiang_queue_backlog:2026091512"), any());
}
@Test
@@ -248,6 +275,95 @@ class MaixiangAnomalyScannerTest {
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
// ======================== 采集机掉线(机器级判活) ========================
@Test
void collectorSilentPushesWhenMachineStaysSilent() {
// 整机停摆:机器级静默 11 分钟 → 一条「采集机掉线」,而不是几十条任务失败
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(700L, List.of(
new WorkerItem("pt", "跟价", 1_790_000_000L, 700L),
new WorkerItem("vc", "变体采集", 1_790_000_040L, 660L),
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 700L))));
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
.thenReturn(1);
newScanner().checkCollectorSilent(audience, NOW);
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:采集机掉线"),
content.capture(), eq("maixiang_silent:2026091512"), isNull());
assertThat(content.getValue())
.contains("静默 11 分钟")
.contains("跟价 静默 11 分钟")
.contains("cookie 通道")
.contains("无开机自启");
}
@Test
void collectorSilentIgnoresStaleSignalWhileMachineAlive() {
// 核心回归(2026-09-29 实测发现的坑):变体采集领到长任务后不再调 /batch/task/get,
// 于是 vc 信号陈旧,但机器明显活着(ck 0 秒前刚打点)→ 机器级静默 0,绝不能报掉线。
// 服务端把「所有信号里最新的一个」当机器级判活值就是为了这个场景。
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(0L, List.of(
new WorkerItem("pt", "跟价", 1_790_000_000L, 5L),
new WorkerItem("vc", "变体采集", 1_780_000_000L, 36_000L),
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 0L))));
newScanner().checkCollectorSilent(audience, NOW);
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
@Test
void collectorSilentIgnoresFreshMachine() {
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(599L, List.of(
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 599L))));
newScanner().checkCollectorSilent(audience, NOW);
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
@Test
void collectorSilentSkipsWhenNeverReported() {
// machineIdleSeconds=null = 从未上报(刚上线/清过 Redis/接口刚部署)→ 跳过而不是误报掉线;
// 这类情况由「队列积压 / 任务滞留」覆盖
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(null, List.of(
new WorkerItem("pt", "跟价", null, null))));
newScanner().checkCollectorSilent(audience, NOW);
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
@Test
void collectorSilentShowsNeverReportedSignalInContent() {
// 告警文案要能区分「某信号从未上报」与「静默 N 分钟」,否则排查时看不出是哪一路缺
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(900L, List.of(
new WorkerItem("pt", "跟价", 1_790_000_000L, 900L),
new WorkerItem("vc", "变体采集", null, null))));
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
.thenReturn(1);
newScanner().checkCollectorSilent(audience, NOW);
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(),
content.capture(), anyString(), any());
assertThat(content.getValue()).contains("跟价 静默 15 分钟").contains("变体采集 (从未上报)");
}
@Test
void collectorSilentSkipsWhenEndpointUnavailable() {
// 未部署 /api/console/workers(空快照):跳过而不是误报掉线
when(consoleClient.workers()).thenReturn(WorkerSnapshot.EMPTY);
newScanner().checkCollectorSilent(audience, NOW);
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
// ======================== 容错 ========================
@Test