feat(通知): 新增「采集机掉线」告警,并把队列积压判据改成「有活没人干」
背景(2026-09-29):麦象采集机 15:19 起整机停摆 6 小时(5 个 worker 全没,机器
无开机自启),现有告警完全没发现 —— 五类检查全靠「任务 update_time 不新鲜」间接
推断,一次停摆被摊成几十条"任务失败";队列积压又卡在 pending≥300 的阈值上
(当时停在 210),用户看到的是大面积业务失败,误导排查方向。
- 新增第 6 项检查「采集机掉线」:读 18960 新接口 /api/console/workers 的
machineIdleSeconds(采集端打点,见 maixiang_run_task 同批提交),超阈值(默认 10
分钟)推一条管理员告警,文案含逐信号静默时长便于定位。
* 按**机器级**判活而不是逐 worker:变体采集领到长任务后不再调 /batch/task/get,
逐 worker 判会在长任务正常运行时误报「vc 掉线」(实测踩到)。
* machineIdleSeconds=null(从未上报)视为未知并跳过,避免刚上线误报一次。
- 队列积压判据改为 pending≥阈值 **且 processing=0**,阈值默认 300→100:
只看 pending 两边都不讨好(低了误报"一时来不及",高了真出事也不报)。
单 worker 死掉(机器还活着)由这条覆盖。
- 配置:新增 AIIMAGE_NOTIFICATION_MAIXIANG_SILENT_MINUTES(默认 10)。
This commit is contained in:
@@ -59,12 +59,26 @@ public class NotificationProperties {
|
||||
/** 麦象任务失败告警阈值(条):近 30 分钟窗口内失败数达到该值才告警。 */
|
||||
private int maixiangFailMinCount = 1;
|
||||
|
||||
/** 麦象队列积压阈值(条):task:queue 待处理数达到该值告警。 */
|
||||
private int maixiangQueuePendingThreshold = 300;
|
||||
/**
|
||||
* 麦象队列积压阈值(条):task:queue 待处理数达到该值**且无人消费**(processing=0)才告警。
|
||||
*
|
||||
* <p>默认从 300 下调到 100:2026-09-29 采集机整机停摆时队列停在 210 条,300 的阈值全程没触发。
|
||||
* 单靠调低阈值会引入误报,所以判据同时要求 processing=0 ——「有活没人干」才是故障信号。
|
||||
*/
|
||||
private int maixiangQueuePendingThreshold = 100;
|
||||
|
||||
/** 麦象队列积压阈值(条):task:processing 处理中数达到该值告警。 */
|
||||
private int maixiangQueueProcessingThreshold = 100;
|
||||
|
||||
/**
|
||||
* 麦象采集端静默阈值(分钟):18960 记录的各 worker 最后轮询时间超过该时长即视为采集机掉线。
|
||||
*
|
||||
* <p>这是唯一能直接反映「采集机整机停摆」的信号——其余检查都靠任务 update_time 不新鲜
|
||||
* 间接推断,一次停摆会被摊成几十条"任务失败",误导排查方向(2026-09-29 停摆 6 小时即如此)。
|
||||
* 采集端 pt/vc 每 3~10 秒轮询一次,因此 10 分钟足够宽松,不会误报空闲。
|
||||
*/
|
||||
private int maixiangSilentMinutes = 10;
|
||||
|
||||
/** 已读通知保留天数(超期自动清理),默认 90 天。 */
|
||||
private int readRetentionDays = 90;
|
||||
|
||||
|
||||
+44
@@ -92,6 +92,16 @@ public class MaixiangConsoleClient {
|
||||
return parseQueueStates(fetch(buildUrl("/api/console/queue/status"), "队列状态"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 采集端打点快照:逐信号活动时间 + 机器级静默秒数。
|
||||
*
|
||||
* <p>「空快照」(machineIdleSeconds=null)有两种含义(接口未部署 / 从未有人调过采集端
|
||||
* 会调的接口),调用方应跳过掉线判断而不是据此报掉线——否则刚上线时必然误报一次。
|
||||
*/
|
||||
public WorkerSnapshot workers() {
|
||||
return parseWorkers(fetch(buildUrl("/api/console/workers"), "采集端打点"));
|
||||
}
|
||||
|
||||
/** 批量任务结果行数:未注册结果表(table=null)返回 {@link #RESULT_TOTAL_UNSUPPORTED}。 */
|
||||
public long batchResultTotal(String taskId) {
|
||||
String url = buildUrl("/api/console/batch/detail", "task_id=" + taskId, "page=1", "page_size=1");
|
||||
@@ -212,6 +222,26 @@ public class MaixiangConsoleClient {
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* 采集端打点:idleSeconds 为 null 表示「该信号从未上报」(≠ 上报过但已陈旧)。
|
||||
* machineIdleSeconds 由服务端按「所有信号里最新的那一个」算好,客户端不再自行聚合。
|
||||
*/
|
||||
WorkerSnapshot parseWorkers(JsonNode data) {
|
||||
if (data == null || data.isMissingNode()) {
|
||||
return WorkerSnapshot.EMPTY;
|
||||
}
|
||||
List<WorkerItem> items = new ArrayList<>();
|
||||
for (JsonNode node : data.path("workers")) {
|
||||
items.add(new WorkerItem(
|
||||
node.path("signal").asText(""),
|
||||
node.path("label").asText(""),
|
||||
node.path("lastSeenAt").isNumber() ? node.path("lastSeenAt").asLong() : null,
|
||||
node.path("idleSeconds").isNumber() ? node.path("idleSeconds").asLong() : null));
|
||||
}
|
||||
JsonNode machineIdle = data.path("machineIdleSeconds");
|
||||
return new WorkerSnapshot(machineIdle.isNumber() ? machineIdle.asLong() : null, items);
|
||||
}
|
||||
|
||||
long parseBatchResultTotal(JsonNode data) {
|
||||
if (data == null || data.isMissingNode()) {
|
||||
return RESULT_TOTAL_UNSUPPORTED;
|
||||
@@ -264,4 +294,18 @@ public class MaixiangConsoleClient {
|
||||
/** 队列状态一行。 */
|
||||
public record QueueStateItem(int level, long pending, long processing) {
|
||||
}
|
||||
|
||||
/** 采集端打点的一路信号;{@code idleSeconds}={@code null} 表示该信号从未上报过。 */
|
||||
public record WorkerItem(String signal, String label, Long lastSeenEpoch, Long idleSeconds) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 采集端打点快照。
|
||||
*
|
||||
* @param machineIdleSeconds 机器级静默秒数(服务端按所有信号里最新的一个算);null=从未上报
|
||||
* @param workers 逐信号明细,仅用于告警文案里的诊断信息
|
||||
*/
|
||||
public record WorkerSnapshot(Long machineIdleSeconds, List<WorkerItem> workers) {
|
||||
public static final WorkerSnapshot EMPTY = new WorkerSnapshot(null, List.of());
|
||||
}
|
||||
}
|
||||
|
||||
+73
-8
@@ -9,6 +9,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -19,19 +21,26 @@ import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 麦象(18960 任务调度)异常扫描:把「采集端出了事但没人知道」的情况推给管理员。
|
||||
*
|
||||
* <p>五类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数):
|
||||
* <p>六类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数):
|
||||
* <ol>
|
||||
* <li>采集机掉线:采集端打点的机器级静默超龄 —— 唯一能直接反映「整机停摆」的信号;</li>
|
||||
* <li>批量任务停滞:status=0/1 且超过阈值无更新(如消费端 worker 掉线中断采集);</li>
|
||||
* <li>单任务滞留:创建超过阈值仍未完成(跟价等单任务无人消费);</li>
|
||||
* <li>任务失败:近窗口内 status=3 的单任务达到最小条数;</li>
|
||||
* <li>批量任务空结果:已完成但结果 0 条(worker 中途报错时批次仍会被标"已完成");</li>
|
||||
* <li>队列积压:task:queue / task:processing 超过阈值。</li>
|
||||
* <li>队列积压:task:queue 堆积且无人消费 / task:processing 超过阈值。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>「采集端静默」为什么要单列:其余五类都靠「任务 update_time 不新鲜」间接推断,
|
||||
* 一次整机停摆会被摊成几十条"任务失败",用户和运维看到的是大面积业务失败而不是
|
||||
* "采集机掉线"(2026-09-29 停摆 6 小时即如此,且当时 210 条 pending 没到 300 的阈值,
|
||||
* 连积压检查都没触发)。
|
||||
*
|
||||
* <p>全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键:
|
||||
* 停滞类按天、失败/积压按小时、空结果按任务——同一异常窗口内只提醒一次,
|
||||
* 情况变化(新增停滞任务/失败数增长)时刷新内容并重新提醒。
|
||||
@@ -89,6 +98,7 @@ public class MaixiangAnomalyScanner {
|
||||
}
|
||||
NotificationDispatchService.AdminAudience audience = notificationDispatchService.prepareAdminAudience();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
runCheck("采集端静默", () -> checkCollectorSilent(audience, now));
|
||||
runCheck("批量任务停滞", () -> checkStuckBatchTasks(audience, now));
|
||||
runCheck("单任务滞留", () -> checkStuckSingleTasks(audience, now));
|
||||
runCheck("任务失败", () -> checkFailedSingleTasks(audience, now));
|
||||
@@ -104,6 +114,51 @@ public class MaixiangAnomalyScanner {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 采集机掉线:采集端打点的**机器级**静默超过阈值(默认 10 分钟)→ 一条告警。
|
||||
*
|
||||
* <p>为什么按机器级而不是逐 worker:变体采集消费端一旦领到长任务,处理期间就不会再调
|
||||
* /batch/task/get(消费线程即工作线程),逐 worker 判活会在长任务正常运行时误报「vc 掉线」。
|
||||
* 服务端因此把「所有信号里最新的那一个」当机器级判活值 —— cookie 生成器每国 60 秒写回一次,
|
||||
* 不受任务负载影响。
|
||||
*
|
||||
* <p>单个 worker 死掉(机器还活着)不在这里告警:那属于「有活没人干」,
|
||||
* 由队列积压 / 任务滞留检查覆盖。
|
||||
*
|
||||
* <p>{@code machineIdleSeconds=null} 表示从未上报(刚上线 / 清过 Redis),跳过而不是报掉线。
|
||||
*/
|
||||
void checkCollectorSilent(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
||||
int thresholdMinutes = Math.max(1, properties.getMaixiangSilentMinutes());
|
||||
long thresholdSeconds = thresholdMinutes * 60L;
|
||||
WorkerSnapshot snapshot = consoleClient.workers();
|
||||
Long machineIdle = snapshot.machineIdleSeconds();
|
||||
if (machineIdle == null) {
|
||||
log.info("[maixiang-scan] 采集端打点为空(接口未部署或从未上报),跳过掉线检查");
|
||||
return;
|
||||
}
|
||||
if (machineIdle < thresholdSeconds) {
|
||||
log.info("[maixiang-scan] 采集端在报点(机器级静默 {} 秒,阈值 {} 分钟)", machineIdle, thresholdMinutes);
|
||||
return;
|
||||
}
|
||||
String detail = snapshot.workers().stream()
|
||||
.map(worker -> labelOfWorker(worker) + " " + idleText(worker.idleSeconds()))
|
||||
.collect(Collectors.joining("、"));
|
||||
String content = "麦象采集机疑似掉线:所有打点信号已静默 " + (machineIdle / 60) + " 分钟(" + detail
|
||||
+ ")。该机器无开机自启,重启/升配后必须人工拉起;请检查 5 个 worker 是否在运行"
|
||||
+ "(start_task.bat 或 更新并重启.bat)。";
|
||||
push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(HOUR_FORMAT));
|
||||
}
|
||||
|
||||
/** 打点信号的中文标签:接口没给 label 时回退信号名。 */
|
||||
private String labelOfWorker(WorkerItem worker) {
|
||||
return worker.label() == null || worker.label().isBlank() ? worker.signal() : worker.label();
|
||||
}
|
||||
|
||||
/** 静默时长文案;null = 该信号从未上报。 */
|
||||
private String idleText(Long idleSeconds) {
|
||||
return idleSeconds == null ? "(从未上报)" : "静默 " + (idleSeconds / 60) + " 分钟";
|
||||
}
|
||||
|
||||
/** 批量任务停滞:最后更新早于阈值、仍处 status=0/1(服务端按 update_time 过滤后取回)。 */
|
||||
void checkStuckBatchTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
||||
int thresholdMinutes = Math.max(1, properties.getMaixiangStuckMinutes());
|
||||
@@ -200,15 +255,22 @@ public class MaixiangAnomalyScanner {
|
||||
}
|
||||
}
|
||||
|
||||
/** 队列积压:task:queue 待处理 / task:processing 处理中超过阈值。 */
|
||||
/**
|
||||
* 队列积压:待处理堆积且**无人消费**(processing=0),或处理中数量异常高。
|
||||
*
|
||||
* <p>只看 pending 阈值两边都不讨好:阈值低了会把「采集端正常、只是一时来不及」报成异常,
|
||||
* 阈值高了真出事也不报(2026-09-29 停在 210 条 pending,300 的阈值全程没触发)。
|
||||
* 加上「processing=0」这个必要条件后判据变成「有活没人干」——采集端活着时 pt/vc
|
||||
* 每 3~10 秒轮询一次,正在消费的队列 processing 基本不会为 0。
|
||||
*/
|
||||
void checkQueueBacklog(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
||||
int pendingThreshold = Math.max(1, properties.getMaixiangQueuePendingThreshold());
|
||||
int processingThreshold = Math.max(1, properties.getMaixiangQueueProcessingThreshold());
|
||||
StringBuilder detail = new StringBuilder();
|
||||
for (QueueStateItem queue : consoleClient.queueStatus()) {
|
||||
boolean pendingHit = queue.pending() >= pendingThreshold;
|
||||
boolean processingHit = queue.processing() >= processingThreshold;
|
||||
if (!pendingHit && !processingHit) {
|
||||
boolean pendingStuck = queue.pending() >= pendingThreshold && queue.processing() == 0;
|
||||
boolean processingStuck = queue.processing() >= processingThreshold;
|
||||
if (!pendingStuck && !processingStuck) {
|
||||
continue;
|
||||
}
|
||||
if (detail.length() > 0) {
|
||||
@@ -217,13 +279,16 @@ public class MaixiangAnomalyScanner {
|
||||
detail.append("L").append(queue.level())
|
||||
.append(" 待处理 ").append(queue.pending())
|
||||
.append(" / 处理中 ").append(queue.processing());
|
||||
if (pendingStuck) {
|
||||
detail.append("(堆积且无人在消费)");
|
||||
}
|
||||
}
|
||||
if (detail.length() == 0) {
|
||||
log.info("[maixiang-scan] 无队列积压(阈值 待处理{} / 处理中{})", pendingThreshold, processingThreshold);
|
||||
log.info("[maixiang-scan] 无队列积压(阈值 待处理{} 且无消费 / 处理中{})", pendingThreshold, processingThreshold);
|
||||
return;
|
||||
}
|
||||
String content = "麦象单任务队列积压:" + detail + "(阈值 待处理 " + pendingThreshold
|
||||
+ " / 处理中 " + processingThreshold + ")。消费端可能掉线或处理变慢,请尽快排查。";
|
||||
+ " 且无消费 / 处理中 " + processingThreshold + ")。消费端可能掉线,请尽快排查采集机 worker。";
|
||||
push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(HOUR_FORMAT));
|
||||
}
|
||||
|
||||
|
||||
@@ -438,8 +438,10 @@ aiimage:
|
||||
maixiang-stuck-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_STUCK_MINUTES:60}
|
||||
maixiang-single-stuck-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_SINGLE_STUCK_MINUTES:30}
|
||||
maixiang-fail-min-count: ${AIIMAGE_NOTIFICATION_MAIXIANG_FAIL_MIN_COUNT:1}
|
||||
maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:300}
|
||||
maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:100}
|
||||
maixiang-queue-processing-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PROCESSING_THRESHOLD:100}
|
||||
# 采集端 worker 静默阈值(分钟):超时无轮询 = 采集机掉线。唯一直接反映整机停摆的信号。
|
||||
maixiang-silent-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_SILENT_MINUTES:10}
|
||||
read-retention-days: ${AIIMAGE_NOTIFICATION_READ_RETENTION_DAYS:90}
|
||||
# 未读通知也设保留期(此前永不清理,不看铃铛的用户会无限累积);给得比已读宽一倍
|
||||
unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180}
|
||||
|
||||
+34
@@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
@@ -86,6 +87,39 @@ class MaixiangConsoleClientTest {
|
||||
assertThat(queues.get(0).processing()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseWorkersReadsMachineIdleAndPerSignalIdle() {
|
||||
WorkerSnapshot empty = client.parseWorkers(
|
||||
client.parseEnvelope("{\"code\":1,\"mes\":\"成功\",\"data\":{}}", "test"));
|
||||
assertThat(empty.machineIdleSeconds()).isNull();
|
||||
assertThat(empty.workers()).isEmpty();
|
||||
|
||||
WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
|
||||
"{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":12,\"machineLastSeenAt\":1790000000,"
|
||||
+ "\"workers\":["
|
||||
+ "{\"signal\":\"pt\",\"label\":\"跟价\",\"lastSeenAt\":1790000000,\"idleSeconds\":12},"
|
||||
+ "{\"signal\":\"vc\",\"label\":\"变体采集\",\"lastSeenAt\":null,\"idleSeconds\":null}]}}",
|
||||
"test"));
|
||||
|
||||
assertThat(snapshot.machineIdleSeconds()).isEqualTo(12L);
|
||||
assertThat(snapshot.workers()).hasSize(2);
|
||||
assertThat(snapshot.workers().get(0).signal()).isEqualTo("pt");
|
||||
assertThat(snapshot.workers().get(0).label()).isEqualTo("跟价");
|
||||
assertThat(snapshot.workers().get(0).idleSeconds()).isEqualTo(12L);
|
||||
// JSON null 必须解析成 null 而不是 0:0 会被当成「刚刚还在打点」,把掉线判成正常
|
||||
assertThat(snapshot.workers().get(1).idleSeconds()).isNull();
|
||||
assertThat(snapshot.workers().get(1).lastSeenEpoch()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseWorkersTreatsNullMachineIdleAsUnknown() {
|
||||
// 机器级静默为 null = 从未上报(≠ 0 秒)——调用方据此跳过掉线判断,避免刚上线误报
|
||||
WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
|
||||
"{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":null,\"workers\":[]}}", "test"));
|
||||
|
||||
assertThat(snapshot.machineIdleSeconds()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseBatchResultTotalDistinguishesUnsupportedTable() {
|
||||
// 已注册结果表:返回真实行数
|
||||
|
||||
+119
-3
@@ -8,6 +8,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
@@ -225,8 +227,9 @@ class MaixiangAnomalyScannerTest {
|
||||
// ======================== 队列积压 ========================
|
||||
|
||||
@Test
|
||||
void queueBacklogPushesWhenOverThreshold() {
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 500, 3)));
|
||||
void queueBacklogPushesWhenPendingPiledUpAndNobodyConsuming() {
|
||||
// 有活没人干:210 条堆积 + processing=0(2026-09-29 停摆时的真实形态,当时 300 阈值没触发)
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 210, 0)));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
@@ -236,7 +239,31 @@ class MaixiangAnomalyScannerTest {
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"),
|
||||
content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull());
|
||||
assertThat(content.getValue()).contains("L1 待处理 500 / 处理中 3");
|
||||
assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queueBacklogIgnoresPendingWhenConsumerIsWorking() {
|
||||
// 回归守卫:501 条堆积但处理中有 3 个 → 采集端在正常消费,不该报异常
|
||||
// (旧判据只看 pending 阈值,会把这种情况误报成故障)
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 501, 3)));
|
||||
|
||||
newScanner().checkQueueBacklog(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queueBacklogPushesWhenProcessingPiledUp() {
|
||||
// processing 超阈值 = 领走了却不 ACK,仍有独立意义(与 pending 分支互不干扰)
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 0, 150)));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScanner().checkQueueBacklog(audience, NOW);
|
||||
|
||||
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(),
|
||||
eq("maixiang_queue_backlog:2026091512"), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -248,6 +275,95 @@ class MaixiangAnomalyScannerTest {
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
// ======================== 采集机掉线(机器级判活) ========================
|
||||
|
||||
@Test
|
||||
void collectorSilentPushesWhenMachineStaysSilent() {
|
||||
// 整机停摆:机器级静默 11 分钟 → 一条「采集机掉线」,而不是几十条任务失败
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(700L, List.of(
|
||||
new WorkerItem("pt", "跟价", 1_790_000_000L, 700L),
|
||||
new WorkerItem("vc", "变体采集", 1_790_000_040L, 660L),
|
||||
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 700L))));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:采集机掉线"),
|
||||
content.capture(), eq("maixiang_silent:2026091512"), isNull());
|
||||
assertThat(content.getValue())
|
||||
.contains("静默 11 分钟")
|
||||
.contains("跟价 静默 11 分钟")
|
||||
.contains("cookie 通道")
|
||||
.contains("无开机自启");
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentIgnoresStaleSignalWhileMachineAlive() {
|
||||
// 核心回归(2026-09-29 实测发现的坑):变体采集领到长任务后不再调 /batch/task/get,
|
||||
// 于是 vc 信号陈旧,但机器明显活着(ck 0 秒前刚打点)→ 机器级静默 0,绝不能报掉线。
|
||||
// 服务端把「所有信号里最新的一个」当机器级判活值就是为了这个场景。
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(0L, List.of(
|
||||
new WorkerItem("pt", "跟价", 1_790_000_000L, 5L),
|
||||
new WorkerItem("vc", "变体采集", 1_780_000_000L, 36_000L),
|
||||
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 0L))));
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentIgnoresFreshMachine() {
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(599L, List.of(
|
||||
new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 599L))));
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentSkipsWhenNeverReported() {
|
||||
// machineIdleSeconds=null = 从未上报(刚上线/清过 Redis/接口刚部署)→ 跳过而不是误报掉线;
|
||||
// 这类情况由「队列积压 / 任务滞留」覆盖
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(null, List.of(
|
||||
new WorkerItem("pt", "跟价", null, null))));
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentShowsNeverReportedSignalInContent() {
|
||||
// 告警文案要能区分「某信号从未上报」与「静默 N 分钟」,否则排查时看不出是哪一路缺
|
||||
when(consoleClient.workers()).thenReturn(new WorkerSnapshot(900L, List.of(
|
||||
new WorkerItem("pt", "跟价", 1_790_000_000L, 900L),
|
||||
new WorkerItem("vc", "变体采集", null, null))));
|
||||
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(),
|
||||
content.capture(), anyString(), any());
|
||||
assertThat(content.getValue()).contains("跟价 静默 15 分钟").contains("变体采集 (从未上报)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectorSilentSkipsWhenEndpointUnavailable() {
|
||||
// 未部署 /api/console/workers(空快照):跳过而不是误报掉线
|
||||
when(consoleClient.workers()).thenReturn(WorkerSnapshot.EMPTY);
|
||||
|
||||
newScanner().checkCollectorSilent(audience, NOW);
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
// ======================== 容错 ========================
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user