diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java index 61a10f791..881a88c6a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java @@ -59,12 +59,26 @@ public class NotificationProperties { /** 麦象任务失败告警阈值(条):近 30 分钟窗口内失败数达到该值才告警。 */ private int maixiangFailMinCount = 1; - /** 麦象队列积压阈值(条):task:queue 待处理数达到该值告警。 */ - private int maixiangQueuePendingThreshold = 300; + /** + * 麦象队列积压阈值(条):task:queue 待处理数达到该值**且无人消费**(processing=0)才告警。 + * + *

默认从 300 下调到 100:2026-09-29 采集机整机停摆时队列停在 210 条,300 的阈值全程没触发。 + * 单靠调低阈值会引入误报,所以判据同时要求 processing=0 ——「有活没人干」才是故障信号。 + */ + private int maixiangQueuePendingThreshold = 100; /** 麦象队列积压阈值(条):task:processing 处理中数达到该值告警。 */ private int maixiangQueueProcessingThreshold = 100; + /** + * 麦象采集端静默阈值(分钟):18960 记录的各 worker 最后轮询时间超过该时长即视为采集机掉线。 + * + *

这是唯一能直接反映「采集机整机停摆」的信号——其余检查都靠任务 update_time 不新鲜 + * 间接推断,一次停摆会被摊成几十条"任务失败",误导排查方向(2026-09-29 停摆 6 小时即如此)。 + * 采集端 pt/vc 每 3~10 秒轮询一次,因此 10 分钟足够宽松,不会误报空闲。 + */ + private int maixiangSilentMinutes = 10; + /** 已读通知保留天数(超期自动清理),默认 90 天。 */ private int readRetentionDays = 90; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java index fba019df3..33659dc88 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java @@ -92,6 +92,16 @@ public class MaixiangConsoleClient { return parseQueueStates(fetch(buildUrl("/api/console/queue/status"), "队列状态")); } + /** + * 采集端打点快照:逐信号活动时间 + 机器级静默秒数。 + * + *

「空快照」(machineIdleSeconds=null)有两种含义(接口未部署 / 从未有人调过采集端 + * 会调的接口),调用方应跳过掉线判断而不是据此报掉线——否则刚上线时必然误报一次。 + */ + public WorkerSnapshot workers() { + return parseWorkers(fetch(buildUrl("/api/console/workers"), "采集端打点")); + } + /** 批量任务结果行数:未注册结果表(table=null)返回 {@link #RESULT_TOTAL_UNSUPPORTED}。 */ public long batchResultTotal(String taskId) { String url = buildUrl("/api/console/batch/detail", "task_id=" + taskId, "page=1", "page_size=1"); @@ -212,6 +222,26 @@ public class MaixiangConsoleClient { return items; } + /** + * 采集端打点:idleSeconds 为 null 表示「该信号从未上报」(≠ 上报过但已陈旧)。 + * machineIdleSeconds 由服务端按「所有信号里最新的那一个」算好,客户端不再自行聚合。 + */ + WorkerSnapshot parseWorkers(JsonNode data) { + if (data == null || data.isMissingNode()) { + return WorkerSnapshot.EMPTY; + } + List items = new ArrayList<>(); + for (JsonNode node : data.path("workers")) { + items.add(new WorkerItem( + node.path("signal").asText(""), + node.path("label").asText(""), + node.path("lastSeenAt").isNumber() ? node.path("lastSeenAt").asLong() : null, + node.path("idleSeconds").isNumber() ? node.path("idleSeconds").asLong() : null)); + } + JsonNode machineIdle = data.path("machineIdleSeconds"); + return new WorkerSnapshot(machineIdle.isNumber() ? machineIdle.asLong() : null, items); + } + long parseBatchResultTotal(JsonNode data) { if (data == null || data.isMissingNode()) { return RESULT_TOTAL_UNSUPPORTED; @@ -264,4 +294,18 @@ public class MaixiangConsoleClient { /** 队列状态一行。 */ public record QueueStateItem(int level, long pending, long processing) { } + + /** 采集端打点的一路信号;{@code idleSeconds}={@code null} 表示该信号从未上报过。 */ + public record WorkerItem(String signal, String label, Long lastSeenEpoch, Long idleSeconds) { + } + + /** + * 采集端打点快照。 + * + * @param machineIdleSeconds 机器级静默秒数(服务端按所有信号里最新的一个算);null=从未上报 + * @param workers 逐信号明细,仅用于告警文案里的诊断信息 + */ + public record WorkerSnapshot(Long machineIdleSeconds, List workers) { + public static final WorkerSnapshot EMPTY = new WorkerSnapshot(null, List.of()); + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScanner.java b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScanner.java index 33c19ab1b..d2ef49bdc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScanner.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScanner.java @@ -9,6 +9,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage; +import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem; +import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -19,19 +21,26 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * 麦象(18960 任务调度)异常扫描:把「采集端出了事但没人知道」的情况推给管理员。 * - *

五类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数): + *

六类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数): *

    + *
  1. 采集机掉线:采集端打点的机器级静默超龄 —— 唯一能直接反映「整机停摆」的信号;
  2. *
  3. 批量任务停滞:status=0/1 且超过阈值无更新(如消费端 worker 掉线中断采集);
  4. *
  5. 单任务滞留:创建超过阈值仍未完成(跟价等单任务无人消费);
  6. *
  7. 任务失败:近窗口内 status=3 的单任务达到最小条数;
  8. *
  9. 批量任务空结果:已完成但结果 0 条(worker 中途报错时批次仍会被标"已完成");
  10. - *
  11. 队列积压:task:queue / task:processing 超过阈值。
  12. + *
  13. 队列积压:task:queue 堆积且无人消费 / task:processing 超过阈值。
  14. *
* + *

「采集端静默」为什么要单列:其余五类都靠「任务 update_time 不新鲜」间接推断, + * 一次整机停摆会被摊成几十条"任务失败",用户和运维看到的是大面积业务失败而不是 + * "采集机掉线"(2026-09-29 停摆 6 小时即如此,且当时 210 条 pending 没到 300 的阈值, + * 连积压检查都没触发)。 + * *

全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键: * 停滞类按天、失败/积压按小时、空结果按任务——同一异常窗口内只提醒一次, * 情况变化(新增停滞任务/失败数增长)时刷新内容并重新提醒。 @@ -89,6 +98,7 @@ public class MaixiangAnomalyScanner { } NotificationDispatchService.AdminAudience audience = notificationDispatchService.prepareAdminAudience(); LocalDateTime now = LocalDateTime.now(); + runCheck("采集端静默", () -> checkCollectorSilent(audience, now)); runCheck("批量任务停滞", () -> checkStuckBatchTasks(audience, now)); runCheck("单任务滞留", () -> checkStuckSingleTasks(audience, now)); runCheck("任务失败", () -> checkFailedSingleTasks(audience, now)); @@ -104,6 +114,51 @@ public class MaixiangAnomalyScanner { } } + /** + * 采集机掉线:采集端打点的**机器级**静默超过阈值(默认 10 分钟)→ 一条告警。 + * + *

为什么按机器级而不是逐 worker:变体采集消费端一旦领到长任务,处理期间就不会再调 + * /batch/task/get(消费线程即工作线程),逐 worker 判活会在长任务正常运行时误报「vc 掉线」。 + * 服务端因此把「所有信号里最新的那一个」当机器级判活值 —— cookie 生成器每国 60 秒写回一次, + * 不受任务负载影响。 + * + *

单个 worker 死掉(机器还活着)不在这里告警:那属于「有活没人干」, + * 由队列积压 / 任务滞留检查覆盖。 + * + *

{@code machineIdleSeconds=null} 表示从未上报(刚上线 / 清过 Redis),跳过而不是报掉线。 + */ + void checkCollectorSilent(NotificationDispatchService.AdminAudience audience, LocalDateTime now) { + int thresholdMinutes = Math.max(1, properties.getMaixiangSilentMinutes()); + long thresholdSeconds = thresholdMinutes * 60L; + WorkerSnapshot snapshot = consoleClient.workers(); + Long machineIdle = snapshot.machineIdleSeconds(); + if (machineIdle == null) { + log.info("[maixiang-scan] 采集端打点为空(接口未部署或从未上报),跳过掉线检查"); + return; + } + if (machineIdle < thresholdSeconds) { + log.info("[maixiang-scan] 采集端在报点(机器级静默 {} 秒,阈值 {} 分钟)", machineIdle, thresholdMinutes); + return; + } + String detail = snapshot.workers().stream() + .map(worker -> labelOfWorker(worker) + " " + idleText(worker.idleSeconds())) + .collect(Collectors.joining("、")); + String content = "麦象采集机疑似掉线:所有打点信号已静默 " + (machineIdle / 60) + " 分钟(" + detail + + ")。该机器无开机自启,重启/升配后必须人工拉起;请检查 5 个 worker 是否在运行" + + "(start_task.bat 或 更新并重启.bat)。"; + push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(HOUR_FORMAT)); + } + + /** 打点信号的中文标签:接口没给 label 时回退信号名。 */ + private String labelOfWorker(WorkerItem worker) { + return worker.label() == null || worker.label().isBlank() ? worker.signal() : worker.label(); + } + + /** 静默时长文案;null = 该信号从未上报。 */ + private String idleText(Long idleSeconds) { + return idleSeconds == null ? "(从未上报)" : "静默 " + (idleSeconds / 60) + " 分钟"; + } + /** 批量任务停滞:最后更新早于阈值、仍处 status=0/1(服务端按 update_time 过滤后取回)。 */ void checkStuckBatchTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) { int thresholdMinutes = Math.max(1, properties.getMaixiangStuckMinutes()); @@ -200,15 +255,22 @@ public class MaixiangAnomalyScanner { } } - /** 队列积压:task:queue 待处理 / task:processing 处理中超过阈值。 */ + /** + * 队列积压:待处理堆积且**无人消费**(processing=0),或处理中数量异常高。 + * + *

只看 pending 阈值两边都不讨好:阈值低了会把「采集端正常、只是一时来不及」报成异常, + * 阈值高了真出事也不报(2026-09-29 停在 210 条 pending,300 的阈值全程没触发)。 + * 加上「processing=0」这个必要条件后判据变成「有活没人干」——采集端活着时 pt/vc + * 每 3~10 秒轮询一次,正在消费的队列 processing 基本不会为 0。 + */ void checkQueueBacklog(NotificationDispatchService.AdminAudience audience, LocalDateTime now) { int pendingThreshold = Math.max(1, properties.getMaixiangQueuePendingThreshold()); int processingThreshold = Math.max(1, properties.getMaixiangQueueProcessingThreshold()); StringBuilder detail = new StringBuilder(); for (QueueStateItem queue : consoleClient.queueStatus()) { - boolean pendingHit = queue.pending() >= pendingThreshold; - boolean processingHit = queue.processing() >= processingThreshold; - if (!pendingHit && !processingHit) { + boolean pendingStuck = queue.pending() >= pendingThreshold && queue.processing() == 0; + boolean processingStuck = queue.processing() >= processingThreshold; + if (!pendingStuck && !processingStuck) { continue; } if (detail.length() > 0) { @@ -217,13 +279,16 @@ public class MaixiangAnomalyScanner { detail.append("L").append(queue.level()) .append(" 待处理 ").append(queue.pending()) .append(" / 处理中 ").append(queue.processing()); + if (pendingStuck) { + detail.append("(堆积且无人在消费)"); + } } if (detail.length() == 0) { - log.info("[maixiang-scan] 无队列积压(阈值 待处理{} / 处理中{})", pendingThreshold, processingThreshold); + log.info("[maixiang-scan] 无队列积压(阈值 待处理{} 且无消费 / 处理中{})", pendingThreshold, processingThreshold); return; } String content = "麦象单任务队列积压:" + detail + "(阈值 待处理 " + pendingThreshold - + " / 处理中 " + processingThreshold + ")。消费端可能掉线或处理变慢,请尽快排查。"; + + " 且无消费 / 处理中 " + processingThreshold + ")。消费端可能掉线,请尽快排查采集机 worker。"; push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(HOUR_FORMAT)); } diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 90eb185bf..8efb359d0 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -438,8 +438,10 @@ aiimage: maixiang-stuck-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_STUCK_MINUTES:60} maixiang-single-stuck-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_SINGLE_STUCK_MINUTES:30} maixiang-fail-min-count: ${AIIMAGE_NOTIFICATION_MAIXIANG_FAIL_MIN_COUNT:1} - maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:300} + maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:100} maixiang-queue-processing-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PROCESSING_THRESHOLD:100} + # 采集端 worker 静默阈值(分钟):超时无轮询 = 采集机掉线。唯一直接反映整机停摆的信号。 + maixiang-silent-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_SILENT_MINUTES:10} read-retention-days: ${AIIMAGE_NOTIFICATION_READ_RETENTION_DAYS:90} # 未读通知也设保留期(此前永不清理,不看铃铛的用户会无限累积);给得比已读宽一倍 unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java index 52396ca51..244fad6fe 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java @@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage; +import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot; import org.junit.jupiter.api.Test; import java.time.LocalDateTime; @@ -86,6 +87,39 @@ class MaixiangConsoleClientTest { assertThat(queues.get(0).processing()).isEqualTo(3); } + @Test + void parseWorkersReadsMachineIdleAndPerSignalIdle() { + WorkerSnapshot empty = client.parseWorkers( + client.parseEnvelope("{\"code\":1,\"mes\":\"成功\",\"data\":{}}", "test")); + assertThat(empty.machineIdleSeconds()).isNull(); + assertThat(empty.workers()).isEmpty(); + + WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope( + "{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":12,\"machineLastSeenAt\":1790000000," + + "\"workers\":[" + + "{\"signal\":\"pt\",\"label\":\"跟价\",\"lastSeenAt\":1790000000,\"idleSeconds\":12}," + + "{\"signal\":\"vc\",\"label\":\"变体采集\",\"lastSeenAt\":null,\"idleSeconds\":null}]}}", + "test")); + + assertThat(snapshot.machineIdleSeconds()).isEqualTo(12L); + assertThat(snapshot.workers()).hasSize(2); + assertThat(snapshot.workers().get(0).signal()).isEqualTo("pt"); + assertThat(snapshot.workers().get(0).label()).isEqualTo("跟价"); + assertThat(snapshot.workers().get(0).idleSeconds()).isEqualTo(12L); + // JSON null 必须解析成 null 而不是 0:0 会被当成「刚刚还在打点」,把掉线判成正常 + assertThat(snapshot.workers().get(1).idleSeconds()).isNull(); + assertThat(snapshot.workers().get(1).lastSeenEpoch()).isNull(); + } + + @Test + void parseWorkersTreatsNullMachineIdleAsUnknown() { + // 机器级静默为 null = 从未上报(≠ 0 秒)——调用方据此跳过掉线判断,避免刚上线误报 + WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope( + "{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":null,\"workers\":[]}}", "test")); + + assertThat(snapshot.machineIdleSeconds()).isNull(); + } + @Test void parseBatchResultTotalDistinguishesUnsupportedTable() { // 已注册结果表:返回真实行数 diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java index deb2fd88c..685e8969d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java @@ -8,6 +8,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem; import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage; +import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem; +import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -225,8 +227,9 @@ class MaixiangAnomalyScannerTest { // ======================== 队列积压 ======================== @Test - void queueBacklogPushesWhenOverThreshold() { - when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 500, 3))); + void queueBacklogPushesWhenPendingPiledUpAndNobodyConsuming() { + // 有活没人干:210 条堆积 + processing=0(2026-09-29 停摆时的真实形态,当时 300 阈值没触发) + when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 210, 0))); when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any())) .thenReturn(1); @@ -236,7 +239,31 @@ class MaixiangAnomalyScannerTest { verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY), eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"), content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull()); - assertThat(content.getValue()).contains("L1 待处理 500 / 处理中 3"); + assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费"); + } + + @Test + void queueBacklogIgnoresPendingWhenConsumerIsWorking() { + // 回归守卫:501 条堆积但处理中有 3 个 → 采集端在正常消费,不该报异常 + // (旧判据只看 pending 阈值,会把这种情况误报成故障) + when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 501, 3))); + + newScanner().checkQueueBacklog(audience, NOW); + + verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()); + } + + @Test + void queueBacklogPushesWhenProcessingPiledUp() { + // processing 超阈值 = 领走了却不 ACK,仍有独立意义(与 pending 分支互不干扰) + when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 0, 150))); + when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any())) + .thenReturn(1); + + newScanner().checkQueueBacklog(audience, NOW); + + verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), + eq("maixiang_queue_backlog:2026091512"), any()); } @Test @@ -248,6 +275,95 @@ class MaixiangAnomalyScannerTest { verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()); } + // ======================== 采集机掉线(机器级判活) ======================== + + @Test + void collectorSilentPushesWhenMachineStaysSilent() { + // 整机停摆:机器级静默 11 分钟 → 一条「采集机掉线」,而不是几十条任务失败 + when(consoleClient.workers()).thenReturn(new WorkerSnapshot(700L, List.of( + new WorkerItem("pt", "跟价", 1_790_000_000L, 700L), + new WorkerItem("vc", "变体采集", 1_790_000_040L, 660L), + new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 700L)))); + when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any())) + .thenReturn(1); + + newScanner().checkCollectorSilent(audience, NOW); + + ArgumentCaptor content = ArgumentCaptor.forClass(String.class); + verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY), + eq(NotificationService.LEVEL_WARNING), eq("麦象异常:采集机掉线"), + content.capture(), eq("maixiang_silent:2026091512"), isNull()); + assertThat(content.getValue()) + .contains("静默 11 分钟") + .contains("跟价 静默 11 分钟") + .contains("cookie 通道") + .contains("无开机自启"); + } + + @Test + void collectorSilentIgnoresStaleSignalWhileMachineAlive() { + // 核心回归(2026-09-29 实测发现的坑):变体采集领到长任务后不再调 /batch/task/get, + // 于是 vc 信号陈旧,但机器明显活着(ck 0 秒前刚打点)→ 机器级静默 0,绝不能报掉线。 + // 服务端把「所有信号里最新的一个」当机器级判活值就是为了这个场景。 + when(consoleClient.workers()).thenReturn(new WorkerSnapshot(0L, List.of( + new WorkerItem("pt", "跟价", 1_790_000_000L, 5L), + new WorkerItem("vc", "变体采集", 1_780_000_000L, 36_000L), + new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 0L)))); + + newScanner().checkCollectorSilent(audience, NOW); + + verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()); + } + + @Test + void collectorSilentIgnoresFreshMachine() { + when(consoleClient.workers()).thenReturn(new WorkerSnapshot(599L, List.of( + new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 599L)))); + + newScanner().checkCollectorSilent(audience, NOW); + + verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()); + } + + @Test + void collectorSilentSkipsWhenNeverReported() { + // machineIdleSeconds=null = 从未上报(刚上线/清过 Redis/接口刚部署)→ 跳过而不是误报掉线; + // 这类情况由「队列积压 / 任务滞留」覆盖 + when(consoleClient.workers()).thenReturn(new WorkerSnapshot(null, List.of( + new WorkerItem("pt", "跟价", null, null)))); + + newScanner().checkCollectorSilent(audience, NOW); + + verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()); + } + + @Test + void collectorSilentShowsNeverReportedSignalInContent() { + // 告警文案要能区分「某信号从未上报」与「静默 N 分钟」,否则排查时看不出是哪一路缺 + when(consoleClient.workers()).thenReturn(new WorkerSnapshot(900L, List.of( + new WorkerItem("pt", "跟价", 1_790_000_000L, 900L), + new WorkerItem("vc", "变体采集", null, null)))); + when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any())) + .thenReturn(1); + + newScanner().checkCollectorSilent(audience, NOW); + + ArgumentCaptor content = ArgumentCaptor.forClass(String.class); + verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), + content.capture(), anyString(), any()); + assertThat(content.getValue()).contains("跟价 静默 15 分钟").contains("变体采集 (从未上报)"); + } + + @Test + void collectorSilentSkipsWhenEndpointUnavailable() { + // 未部署 /api/console/workers(空快照):跳过而不是误报掉线 + when(consoleClient.workers()).thenReturn(WorkerSnapshot.EMPTY); + + newScanner().checkCollectorSilent(audience, NOW); + + verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()); + } + // ======================== 容错 ======================== @Test