diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java index 61a10f791..881a88c6a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/NotificationProperties.java @@ -59,12 +59,26 @@ public class NotificationProperties { /** 麦象任务失败告警阈值(条):近 30 分钟窗口内失败数达到该值才告警。 */ private int maixiangFailMinCount = 1; - /** 麦象队列积压阈值(条):task:queue 待处理数达到该值告警。 */ - private int maixiangQueuePendingThreshold = 300; + /** + * 麦象队列积压阈值(条):task:queue 待处理数达到该值**且无人消费**(processing=0)才告警。 + * + *
默认从 300 下调到 100:2026-09-29 采集机整机停摆时队列停在 210 条,300 的阈值全程没触发。 + * 单靠调低阈值会引入误报,所以判据同时要求 processing=0 ——「有活没人干」才是故障信号。 + */ + private int maixiangQueuePendingThreshold = 100; /** 麦象队列积压阈值(条):task:processing 处理中数达到该值告警。 */ private int maixiangQueueProcessingThreshold = 100; + /** + * 麦象采集端静默阈值(分钟):18960 记录的各 worker 最后轮询时间超过该时长即视为采集机掉线。 + * + *
这是唯一能直接反映「采集机整机停摆」的信号——其余检查都靠任务 update_time 不新鲜 + * 间接推断,一次停摆会被摊成几十条"任务失败",误导排查方向(2026-09-29 停摆 6 小时即如此)。 + * 采集端 pt/vc 每 3~10 秒轮询一次,因此 10 分钟足够宽松,不会误报空闲。 + */ + private int maixiangSilentMinutes = 10; + /** 已读通知保留天数(超期自动清理),默认 90 天。 */ private int readRetentionDays = 90; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java index fba019df3..33659dc88 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClient.java @@ -92,6 +92,16 @@ public class MaixiangConsoleClient { return parseQueueStates(fetch(buildUrl("/api/console/queue/status"), "队列状态")); } + /** + * 采集端打点快照:逐信号活动时间 + 机器级静默秒数。 + * + *
「空快照」(machineIdleSeconds=null)有两种含义(接口未部署 / 从未有人调过采集端
+ * 会调的接口),调用方应跳过掉线判断而不是据此报掉线——否则刚上线时必然误报一次。
+ */
+ public WorkerSnapshot workers() {
+ return parseWorkers(fetch(buildUrl("/api/console/workers"), "采集端打点"));
+ }
+
/** 批量任务结果行数:未注册结果表(table=null)返回 {@link #RESULT_TOTAL_UNSUPPORTED}。 */
public long batchResultTotal(String taskId) {
String url = buildUrl("/api/console/batch/detail", "task_id=" + taskId, "page=1", "page_size=1");
@@ -212,6 +222,26 @@ public class MaixiangConsoleClient {
return items;
}
+ /**
+ * 采集端打点:idleSeconds 为 null 表示「该信号从未上报」(≠ 上报过但已陈旧)。
+ * machineIdleSeconds 由服务端按「所有信号里最新的那一个」算好,客户端不再自行聚合。
+ */
+ WorkerSnapshot parseWorkers(JsonNode data) {
+ if (data == null || data.isMissingNode()) {
+ return WorkerSnapshot.EMPTY;
+ }
+ List 五类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数):
+ * 六类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数):
* 「采集端静默」为什么要单列:其余五类都靠「任务 update_time 不新鲜」间接推断,
+ * 一次整机停摆会被摊成几十条"任务失败",用户和运维看到的是大面积业务失败而不是
+ * "采集机掉线"(2026-09-29 停摆 6 小时即如此,且当时 210 条 pending 没到 300 的阈值,
+ * 连积压检查都没触发)。
+ *
* 全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键:
* 停滞类按天、失败/积压按小时、空结果按任务——同一异常窗口内只提醒一次,
* 情况变化(新增停滞任务/失败数增长)时刷新内容并重新提醒。
@@ -89,6 +98,7 @@ public class MaixiangAnomalyScanner {
}
NotificationDispatchService.AdminAudience audience = notificationDispatchService.prepareAdminAudience();
LocalDateTime now = LocalDateTime.now();
+ runCheck("采集端静默", () -> checkCollectorSilent(audience, now));
runCheck("批量任务停滞", () -> checkStuckBatchTasks(audience, now));
runCheck("单任务滞留", () -> checkStuckSingleTasks(audience, now));
runCheck("任务失败", () -> checkFailedSingleTasks(audience, now));
@@ -104,6 +114,51 @@ public class MaixiangAnomalyScanner {
}
}
+ /**
+ * 采集机掉线:采集端打点的**机器级**静默超过阈值(默认 10 分钟)→ 一条告警。
+ *
+ * 为什么按机器级而不是逐 worker:变体采集消费端一旦领到长任务,处理期间就不会再调
+ * /batch/task/get(消费线程即工作线程),逐 worker 判活会在长任务正常运行时误报「vc 掉线」。
+ * 服务端因此把「所有信号里最新的那一个」当机器级判活值 —— cookie 生成器每国 60 秒写回一次,
+ * 不受任务负载影响。
+ *
+ * 单个 worker 死掉(机器还活着)不在这里告警:那属于「有活没人干」,
+ * 由队列积压 / 任务滞留检查覆盖。
+ *
+ * {@code machineIdleSeconds=null} 表示从未上报(刚上线 / 清过 Redis),跳过而不是报掉线。
+ */
+ void checkCollectorSilent(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
+ int thresholdMinutes = Math.max(1, properties.getMaixiangSilentMinutes());
+ long thresholdSeconds = thresholdMinutes * 60L;
+ WorkerSnapshot snapshot = consoleClient.workers();
+ Long machineIdle = snapshot.machineIdleSeconds();
+ if (machineIdle == null) {
+ log.info("[maixiang-scan] 采集端打点为空(接口未部署或从未上报),跳过掉线检查");
+ return;
+ }
+ if (machineIdle < thresholdSeconds) {
+ log.info("[maixiang-scan] 采集端在报点(机器级静默 {} 秒,阈值 {} 分钟)", machineIdle, thresholdMinutes);
+ return;
+ }
+ String detail = snapshot.workers().stream()
+ .map(worker -> labelOfWorker(worker) + " " + idleText(worker.idleSeconds()))
+ .collect(Collectors.joining("、"));
+ String content = "麦象采集机疑似掉线:所有打点信号已静默 " + (machineIdle / 60) + " 分钟(" + detail
+ + ")。该机器无开机自启,重启/升配后必须人工拉起;请检查 5 个 worker 是否在运行"
+ + "(start_task.bat 或 更新并重启.bat)。";
+ push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(HOUR_FORMAT));
+ }
+
+ /** 打点信号的中文标签:接口没给 label 时回退信号名。 */
+ private String labelOfWorker(WorkerItem worker) {
+ return worker.label() == null || worker.label().isBlank() ? worker.signal() : worker.label();
+ }
+
+ /** 静默时长文案;null = 该信号从未上报。 */
+ private String idleText(Long idleSeconds) {
+ return idleSeconds == null ? "(从未上报)" : "静默 " + (idleSeconds / 60) + " 分钟";
+ }
+
/** 批量任务停滞:最后更新早于阈值、仍处 status=0/1(服务端按 update_time 过滤后取回)。 */
void checkStuckBatchTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
int thresholdMinutes = Math.max(1, properties.getMaixiangStuckMinutes());
@@ -200,15 +255,22 @@ public class MaixiangAnomalyScanner {
}
}
- /** 队列积压:task:queue 待处理 / task:processing 处理中超过阈值。 */
+ /**
+ * 队列积压:待处理堆积且**无人消费**(processing=0),或处理中数量异常高。
+ *
+ * 只看 pending 阈值两边都不讨好:阈值低了会把「采集端正常、只是一时来不及」报成异常,
+ * 阈值高了真出事也不报(2026-09-29 停在 210 条 pending,300 的阈值全程没触发)。
+ * 加上「processing=0」这个必要条件后判据变成「有活没人干」——采集端活着时 pt/vc
+ * 每 3~10 秒轮询一次,正在消费的队列 processing 基本不会为 0。
+ */
void checkQueueBacklog(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
int pendingThreshold = Math.max(1, properties.getMaixiangQueuePendingThreshold());
int processingThreshold = Math.max(1, properties.getMaixiangQueueProcessingThreshold());
StringBuilder detail = new StringBuilder();
for (QueueStateItem queue : consoleClient.queueStatus()) {
- boolean pendingHit = queue.pending() >= pendingThreshold;
- boolean processingHit = queue.processing() >= processingThreshold;
- if (!pendingHit && !processingHit) {
+ boolean pendingStuck = queue.pending() >= pendingThreshold && queue.processing() == 0;
+ boolean processingStuck = queue.processing() >= processingThreshold;
+ if (!pendingStuck && !processingStuck) {
continue;
}
if (detail.length() > 0) {
@@ -217,13 +279,16 @@ public class MaixiangAnomalyScanner {
detail.append("L").append(queue.level())
.append(" 待处理 ").append(queue.pending())
.append(" / 处理中 ").append(queue.processing());
+ if (pendingStuck) {
+ detail.append("(堆积且无人在消费)");
+ }
}
if (detail.length() == 0) {
- log.info("[maixiang-scan] 无队列积压(阈值 待处理{} / 处理中{})", pendingThreshold, processingThreshold);
+ log.info("[maixiang-scan] 无队列积压(阈值 待处理{} 且无消费 / 处理中{})", pendingThreshold, processingThreshold);
return;
}
String content = "麦象单任务队列积压:" + detail + "(阈值 待处理 " + pendingThreshold
- + " / 处理中 " + processingThreshold + ")。消费端可能掉线或处理变慢,请尽快排查。";
+ + " 且无消费 / 处理中 " + processingThreshold + ")。消费端可能掉线,请尽快排查采集机 worker。";
push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(HOUR_FORMAT));
}
diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml
index 90eb185bf..8efb359d0 100644
--- a/backend-java/src/main/resources/application.yml
+++ b/backend-java/src/main/resources/application.yml
@@ -438,8 +438,10 @@ aiimage:
maixiang-stuck-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_STUCK_MINUTES:60}
maixiang-single-stuck-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_SINGLE_STUCK_MINUTES:30}
maixiang-fail-min-count: ${AIIMAGE_NOTIFICATION_MAIXIANG_FAIL_MIN_COUNT:1}
- maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:300}
+ maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:100}
maixiang-queue-processing-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PROCESSING_THRESHOLD:100}
+ # 采集端 worker 静默阈值(分钟):超时无轮询 = 采集机掉线。唯一直接反映整机停摆的信号。
+ maixiang-silent-minutes: ${AIIMAGE_NOTIFICATION_MAIXIANG_SILENT_MINUTES:10}
read-retention-days: ${AIIMAGE_NOTIFICATION_READ_RETENTION_DAYS:90}
# 未读通知也设保留期(此前永不清理,不看铃铛的用户会无限累积);给得比已读宽一倍
unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java
index 52396ca51..244fad6fe 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/client/MaixiangConsoleClientTest.java
@@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
+import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
@@ -86,6 +87,39 @@ class MaixiangConsoleClientTest {
assertThat(queues.get(0).processing()).isEqualTo(3);
}
+ @Test
+ void parseWorkersReadsMachineIdleAndPerSignalIdle() {
+ WorkerSnapshot empty = client.parseWorkers(
+ client.parseEnvelope("{\"code\":1,\"mes\":\"成功\",\"data\":{}}", "test"));
+ assertThat(empty.machineIdleSeconds()).isNull();
+ assertThat(empty.workers()).isEmpty();
+
+ WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
+ "{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":12,\"machineLastSeenAt\":1790000000,"
+ + "\"workers\":["
+ + "{\"signal\":\"pt\",\"label\":\"跟价\",\"lastSeenAt\":1790000000,\"idleSeconds\":12},"
+ + "{\"signal\":\"vc\",\"label\":\"变体采集\",\"lastSeenAt\":null,\"idleSeconds\":null}]}}",
+ "test"));
+
+ assertThat(snapshot.machineIdleSeconds()).isEqualTo(12L);
+ assertThat(snapshot.workers()).hasSize(2);
+ assertThat(snapshot.workers().get(0).signal()).isEqualTo("pt");
+ assertThat(snapshot.workers().get(0).label()).isEqualTo("跟价");
+ assertThat(snapshot.workers().get(0).idleSeconds()).isEqualTo(12L);
+ // JSON null 必须解析成 null 而不是 0:0 会被当成「刚刚还在打点」,把掉线判成正常
+ assertThat(snapshot.workers().get(1).idleSeconds()).isNull();
+ assertThat(snapshot.workers().get(1).lastSeenEpoch()).isNull();
+ }
+
+ @Test
+ void parseWorkersTreatsNullMachineIdleAsUnknown() {
+ // 机器级静默为 null = 从未上报(≠ 0 秒)——调用方据此跳过掉线判断,避免刚上线误报
+ WorkerSnapshot snapshot = client.parseWorkers(client.parseEnvelope(
+ "{\"code\":1,\"mes\":\"成功\",\"data\":{\"machineIdleSeconds\":null,\"workers\":[]}}", "test"));
+
+ assertThat(snapshot.machineIdleSeconds()).isNull();
+ }
+
@Test
void parseBatchResultTotalDistinguishesUnsupportedTable() {
// 已注册结果表:返回真实行数
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java
index deb2fd88c..685e8969d 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/notification/service/MaixiangAnomalyScannerTest.java
@@ -8,6 +8,8 @@ import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.Batch
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
+import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerItem;
+import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -225,8 +227,9 @@ class MaixiangAnomalyScannerTest {
// ======================== 队列积压 ========================
@Test
- void queueBacklogPushesWhenOverThreshold() {
- when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 500, 3)));
+ void queueBacklogPushesWhenPendingPiledUpAndNobodyConsuming() {
+ // 有活没人干:210 条堆积 + processing=0(2026-09-29 停摆时的真实形态,当时 300 阈值没触发)
+ when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 210, 0)));
when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
.thenReturn(1);
@@ -236,7 +239,31 @@ class MaixiangAnomalyScannerTest {
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"),
content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull());
- assertThat(content.getValue()).contains("L1 待处理 500 / 处理中 3");
+ assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费");
+ }
+
+ @Test
+ void queueBacklogIgnoresPendingWhenConsumerIsWorking() {
+ // 回归守卫:501 条堆积但处理中有 3 个 → 采集端在正常消费,不该报异常
+ // (旧判据只看 pending 阈值,会把这种情况误报成故障)
+ when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 501, 3)));
+
+ newScanner().checkQueueBacklog(audience, NOW);
+
+ verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
+ }
+
+ @Test
+ void queueBacklogPushesWhenProcessingPiledUp() {
+ // processing 超阈值 = 领走了却不 ACK,仍有独立意义(与 pending 分支互不干扰)
+ when(consoleClient.queueStatus()).thenReturn(List.of(new QueueStateItem(1, 0, 150)));
+ when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
+ .thenReturn(1);
+
+ newScanner().checkQueueBacklog(audience, NOW);
+
+ verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(),
+ eq("maixiang_queue_backlog:2026091512"), any());
}
@Test
@@ -248,6 +275,95 @@ class MaixiangAnomalyScannerTest {
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
}
+ // ======================== 采集机掉线(机器级判活) ========================
+
+ @Test
+ void collectorSilentPushesWhenMachineStaysSilent() {
+ // 整机停摆:机器级静默 11 分钟 → 一条「采集机掉线」,而不是几十条任务失败
+ when(consoleClient.workers()).thenReturn(new WorkerSnapshot(700L, List.of(
+ new WorkerItem("pt", "跟价", 1_790_000_000L, 700L),
+ new WorkerItem("vc", "变体采集", 1_790_000_040L, 660L),
+ new WorkerItem("ck", "cookie 通道", 1_790_000_000L, 700L))));
+ when(dispatch.pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any()))
+ .thenReturn(1);
+
+ newScanner().checkCollectorSilent(audience, NOW);
+
+ ArgumentCaptor
+ *
*
+ *