fix(通知): 麦象不可读不再折叠成「无异常」、探活 4xx 不冒充正常、交付不完整独立成 scene,前端入队拒绝按机器码决定重试
- MaixiangConsoleClient 新增 probeReadable/classifyReadout,把四种「读不出来」的形状与
「确实无异常」分开;MaixiangAnomalyScanner 在六项扫描前先过这道闸,读不通就喊一条
天级去重的管理员异常并退出本轮——此前后台挂掉时六项一致报「无异常」,服务探测还是绿的。
- NotificationScanScheduler 的探活改三态 ALIVE/UNVERIFIED/DOWN:401/403/404 说明请求被
挡在业务之外,拿不到健康证据,不能再按 <500 判成「服务正常」。
- task_partial(交付不完整)从 task_failed 里拆出来独立成 scene,仍归任务异常大类;
NotificationDispatchService 的管理员行改用 SCENE_TASK_PARTIAL。
- 前端 dispatch-guard 用机器码 shop_busy/queue_full 判定「等一会儿再推就会成」,
暂缓 20s×15 封顶;此前页面靠 includes('当前店铺正在执行中') 匹配中文,而文案中间
插了店铺名,永远匹配不到⇒同店互斥被写成任务失败。
护栏:后端 notification 六支用例(含 MaixiangConsoleClientTest 15 例、
MaixiangAnomalyScannerTest 24 例)、前端 dispatch-guard 与 enqueue-rejection-wiring 两支;
后端全量 3827 例 0 红、前端 717 例 0 失败;变异 M1-M7、M12-M14 共 10/10 杀。
This commit is contained in:
+62
-5
@@ -44,6 +44,37 @@ public class MaixiangConsoleClient {
|
||||
return hasText(properties.getPriceTrackApiUrl()) && hasText(properties.getMaixiangConsoleToken());
|
||||
}
|
||||
|
||||
/**
|
||||
* 后台数据可读性探测:真拉一次队列状态接口(最轻的一个),把「读不到」与「读到但没异常」分开。
|
||||
*
|
||||
* <p>异常扫描的六项检查全靠后台接口取数,读不通时它们只会一致地报「一切正常」;而 HTTP 200 +
|
||||
* code!=1(令牌失效 / 接口未部署 / 契约变更)连服务探测都是绿的 —— 那就是长期的静默失效面。
|
||||
*/
|
||||
public ConsoleReadout probeReadable() {
|
||||
Attempt attempt = attempt(buildUrl("/api/console/queue/status"));
|
||||
return classifyReadout(attempt.body(), attempt.httpError());
|
||||
}
|
||||
|
||||
/** 判定一次取数结果的可读性(包内可见,供单测直接喂四种失败形态的样例)。 */
|
||||
ConsoleReadout classifyReadout(String body, String httpError) {
|
||||
if (httpError != null) {
|
||||
return ConsoleReadout.unreadable("接口请求失败:" + httpError);
|
||||
}
|
||||
if (!hasText(body)) {
|
||||
return ConsoleReadout.unreadable("接口返回空响应");
|
||||
}
|
||||
JsonNode root;
|
||||
try {
|
||||
root = objectMapper.readTree(body);
|
||||
} catch (Exception ex) {
|
||||
return ConsoleReadout.unreadable("响应不是合法 JSON(可能被网关拦截):" + ex.getMessage());
|
||||
}
|
||||
if (root.path("code").asInt(-1) != 1) {
|
||||
return ConsoleReadout.unreadable("接口返回失败:" + root.path("mes").asText(""));
|
||||
}
|
||||
return ConsoleReadout.OK;
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量任务列表(all_task)。updateTimeBefore 非空时只取「最后更新」不晚于该时刻的任务
|
||||
* (SQL 字符串比较)。用 update_time 过滤而不是 create_time:desc(id) 分页下按创建时间过滤
|
||||
@@ -112,22 +143,30 @@ public class MaixiangConsoleClient {
|
||||
|
||||
/** 拉取并展开统一信封 {code,mes,data};失败返回 null(已记日志)。 */
|
||||
private JsonNode fetch(String url, String scene) {
|
||||
String body;
|
||||
Attempt attempt = attempt(url);
|
||||
if (attempt.httpError() != null) {
|
||||
log.warn("[maixiang-console] 请求失败 scene={} err={}", scene, attempt.httpError());
|
||||
return null;
|
||||
}
|
||||
return parseEnvelope(attempt.body(), scene);
|
||||
}
|
||||
|
||||
/** 发一次只读 GET:body 与 httpError 恰有一个非空,异常语义保留给调用方判定。 */
|
||||
private Attempt attempt(String url) {
|
||||
try {
|
||||
// 用 URI 对象提交:URL 已由 buildUrl 编码过一次,不能再经 RestClient 模板再次编码
|
||||
// (字符串形态会被二次编码,%20 变 %2520,服务端 DATETIME 参数直接报错)
|
||||
body = RestClient.builder()
|
||||
String body = RestClient.builder()
|
||||
.requestFactory(HttpClientPool.requestFactory(READ_TIMEOUT_MILLIS))
|
||||
.build()
|
||||
.get()
|
||||
.uri(URI.create(url))
|
||||
.retrieve()
|
||||
.body(String.class);
|
||||
return new Attempt(body, null);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[maixiang-console] 请求失败 scene={} err={}", scene, ex.getMessage());
|
||||
return null;
|
||||
return new Attempt(null, ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
|
||||
}
|
||||
return parseEnvelope(body, scene);
|
||||
}
|
||||
|
||||
/** 组装后台接口 URL(含 token 与查询参数,统一编码一次;包内可见供单测校验编码行为)。 */
|
||||
@@ -272,6 +311,24 @@ public class MaixiangConsoleClient {
|
||||
|
||||
// ======================== 数据模型 ========================
|
||||
|
||||
/**
|
||||
* 后台可读性探测结果。
|
||||
*
|
||||
* @param readable false = 本轮所有后台读数都不可信(≠「没有异常」)
|
||||
* @param reason 不可读原因,含服务端 mes,直接进告警文案
|
||||
*/
|
||||
public record ConsoleReadout(boolean readable, String reason) {
|
||||
public static final ConsoleReadout OK = new ConsoleReadout(true, "");
|
||||
|
||||
public static ConsoleReadout unreadable(String reason) {
|
||||
return new ConsoleReadout(false, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** 一次取数的原始结果:{@code body} 与 {@code httpError} 互斥。 */
|
||||
private record Attempt(String body, String httpError) {
|
||||
}
|
||||
|
||||
/** 批量任务(all_task)单条;{@code resMes} 是进度/结果文字(旧版接口缺失时为空串)。 */
|
||||
public record BatchTaskItem(String taskId, String taskType, String uid, int status,
|
||||
LocalDateTime createTime, LocalDateTime updateTime,
|
||||
|
||||
+15
-1
@@ -6,6 +6,7 @@ import com.nanri.aiimage.config.NotificationProperties;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.ConsoleReadout;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
@@ -26,7 +27,7 @@ import java.util.stream.Collectors;
|
||||
/**
|
||||
* 麦象(18960 任务调度)异常扫描:把「采集端出了事但没人知道」的情况推给管理员。
|
||||
*
|
||||
* <p>六类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数):
|
||||
* <p>一道「后台可读性」闸门 + 六类检查(全部只读,通过 {@link MaixiangConsoleClient} 的后台接口取数):
|
||||
* <ol>
|
||||
* <li>采集机掉线:采集端打点的机器级静默超龄 —— 唯一能直接反映「整机停摆」的信号;</li>
|
||||
* <li>批量任务停滞:status=0/1 且超过阈值无更新(如消费端 worker 掉线中断采集);</li>
|
||||
@@ -36,6 +37,9 @@ import java.util.stream.Collectors;
|
||||
* <li>队列积压:task:queue 堆积且无人消费 / task:processing 超过阈值。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>「后台可读性」闸门为什么要排在最前:六项检查依赖同一批接口,接口整体读不通时它们的一致结果是
|
||||
* 「查无异常」而不是「扫不出」;HTTP 200 + code!=1(令牌失效、接口未部署、契约变更)连服务探测都是绿的。
|
||||
*
|
||||
* <p>「采集端静默」为什么要单列:其余五类都靠「任务 update_time 不新鲜」间接推断,
|
||||
* 一次整机停摆会被摊成几十条"任务失败",用户和运维看到的是大面积业务失败而不是
|
||||
* "采集机掉线"(2026-09-29 停摆 6 小时即如此,且当时 210 条 pending 没到 300 的阈值,
|
||||
@@ -97,6 +101,16 @@ public class MaixiangAnomalyScanner {
|
||||
}
|
||||
NotificationDispatchService.AdminAudience audience = notificationDispatchService.prepareAdminAudience();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
ConsoleReadout readout = consoleClient.probeReadable();
|
||||
if (!readout.readable()) {
|
||||
// 后台读不通时下面六项只会一致地报「无异常」(服务探测甚至还是绿的),先喊出来再退出本轮
|
||||
push(audience, "麦象后台数据不可读",
|
||||
"麦象后台接口取不到数据,本轮异常扫描已跳过(掉线/停滞/滞留/失败/空结果/队列积压六项都不会报,"
|
||||
+ "而 18960 服务探测可能仍是正常)。原因:" + readout.reason()
|
||||
+ "。请检查后台令牌 maixiang-console-token 是否失效、麦象是否升级换了接口契约。",
|
||||
"maixiang_console_unreadable:" + now.format(DAY_FORMAT));
|
||||
return;
|
||||
}
|
||||
runCheck("采集端静默", () -> checkCollectorSilent(audience, now));
|
||||
runCheck("批量任务停滞", () -> checkStuckBatchTasks(audience, now));
|
||||
runCheck("单任务滞留", () -> checkStuckSingleTasks(audience, now));
|
||||
|
||||
+10
-6
@@ -109,9 +109,12 @@ public class NotificationDispatchService {
|
||||
* 「任务成功,但有一部分行没跑出来」通知:整任务没失败、铃铛里的 task_failed 只覆盖终态失败,
|
||||
* 这类部分缺失过去对用户完全不可见(品牌检测的查询失败行只躺在结果文件的一张子表里)。
|
||||
*
|
||||
* <p>场景沿用 {@code task_failed}(大类仍是「任务异常」),去重键按任务维度:同一任务重复
|
||||
* 组装结果时刷新内容而不是刷屏。不写 task.errorMessage —— 结果组装的失败判据读那个字段,
|
||||
* 写了会把任务翻成 FAILED,与「状态保持成功、只提示交付不完整」的产品口径相悖。
|
||||
* <p>场景用独立的 {@code task_partial}(大类仍是「任务异常」,用户看到的分类没变):过去它
|
||||
* 沿用 task_failed,巡检按 task_failed 独立桶数报「窗口内 N 条独立失败」时会把交付不完整一起
|
||||
* 数成任务失败(2026-10-05 值班:窗口 22 行里 20 行是这种),既误报也压住了真失败的读数。
|
||||
* 去重键按任务维度不变:同一任务重复组装结果时刷新内容而不是刷屏。不写 task.errorMessage ——
|
||||
* 结果组装的失败判据读那个字段,写了会把任务翻成 FAILED,与「状态保持成功、只提示交付不完整」
|
||||
* 的产品口径相悖。
|
||||
*/
|
||||
public void pushPartialResult(Long userId, Long taskId, String moduleType, String taskNo, String detail) {
|
||||
if (userId == null || userId <= 0 || taskId == null || detail == null || detail.isBlank()) {
|
||||
@@ -120,10 +123,10 @@ public class NotificationDispatchService {
|
||||
String title = TaskModuleRegistry.labelOf(moduleType) + "任务部分结果缺失";
|
||||
String displayTaskNo = taskNo == null || taskNo.isBlank() ? "#" + taskId : taskNo.trim();
|
||||
notificationService.pushOrRefresh(userId, NotificationService.AUDIENCE_USER,
|
||||
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
|
||||
NotificationService.SCENE_TASK_PARTIAL, NotificationService.LEVEL_WARNING,
|
||||
title, "任务 " + displayTaskNo + ":" + detail + "。任务已完成,其余结果正常,可在结果文件里查看缺失清单。",
|
||||
"task_partial:" + taskId);
|
||||
pushToAdmins(NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
|
||||
pushToAdmins(NotificationService.SCENE_TASK_PARTIAL, NotificationService.LEVEL_WARNING,
|
||||
"用户" + title,
|
||||
"用户 " + displayNameOf(userId) + "(uid=" + userId + ")任务 " + displayTaskNo + ":" + detail,
|
||||
"task_partial_admin:" + taskId, userId);
|
||||
@@ -135,6 +138,7 @@ public class NotificationDispatchService {
|
||||
*
|
||||
* <p>只推用户不推管理员:这是部分缺失(task_partial)的自动修复跟进,管理员侧
|
||||
* 已经在缺失发生时知晓,补测成功属于正向收敛——再扇出一份只放大铃铛噪声。
|
||||
* 场景与缺失那条同为 task_partial:它说的还是同一件「交付不完整」的事,不是任务失败。
|
||||
* 同任务同自然日一条(dedupe_key 带日期):补测按轮推进,同一轮内多品牌合并成一条。
|
||||
*/
|
||||
public void pushRecheckOutcome(Long userId, Long taskId, String moduleType, String taskNo,
|
||||
@@ -165,7 +169,7 @@ public class NotificationDispatchService {
|
||||
String level = fixed && pendingRows <= 0
|
||||
? NotificationService.LEVEL_INFO : NotificationService.LEVEL_WARNING;
|
||||
notificationService.pushOrRefresh(userId, NotificationService.AUDIENCE_USER,
|
||||
NotificationService.SCENE_TASK_FAILED, level, title, content,
|
||||
NotificationService.SCENE_TASK_PARTIAL, level, title, content,
|
||||
"collect_recheck:" + taskId + ":" + java.time.LocalDate.now()
|
||||
.format(java.time.format.DateTimeFormatter.BASIC_ISO_DATE));
|
||||
}
|
||||
|
||||
+56
-11
@@ -200,9 +200,10 @@ public class NotificationScanScheduler {
|
||||
}
|
||||
|
||||
/** HTTP 探测:失败立即重试一次,两次都失败才算故障(过滤瞬时抖动)。 */
|
||||
private ProbeOutcome probeHttpWithRetry(String url) {
|
||||
ProbeOutcome probeHttpWithRetry(String url) {
|
||||
ProbeOutcome first = probeHttpOnce(url);
|
||||
if (first.ok()) {
|
||||
if (first.verdict() != ProbeVerdict.DOWN) {
|
||||
// ALIVE 无需重试;UNVERIFIED 是确定性结果,再打一次只多一份日志
|
||||
return first;
|
||||
}
|
||||
log.warn("[notification-scan] 服务探测第一次失败,重试一次 url={} detail={}", url, first.detail());
|
||||
@@ -219,14 +220,13 @@ public class NotificationScanScheduler {
|
||||
.uri(url)
|
||||
.exchange((request, response) -> response.getStatusCode().value());
|
||||
long latency = System.currentTimeMillis() - startMillis;
|
||||
// 任意 HTTP < 500 视为存活(4xx 说明服务在线,仅路径/鉴权问题)
|
||||
if (status != null && status < 500) {
|
||||
return new ProbeOutcome(true, "HTTP " + status + "(" + latency + "ms)");
|
||||
if (status == null) {
|
||||
return ProbeOutcome.unverified("无响应状态码(" + latency + "ms)");
|
||||
}
|
||||
return new ProbeOutcome(false, "HTTP " + status + "(" + latency + "ms)");
|
||||
return probeOutcomeFor(status, latency);
|
||||
} catch (Exception ex) {
|
||||
long latency = System.currentTimeMillis() - startMillis;
|
||||
return new ProbeOutcome(false, rootCauseMessage(ex) + "(" + latency + "ms)");
|
||||
return ProbeOutcome.down(rootCauseMessage(ex) + "(" + latency + "ms)");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,7 +235,7 @@ public class NotificationScanScheduler {
|
||||
try {
|
||||
ProxyBalancePort.ProxyBalanceSnapshot balance = proxyBalancePort.fetchBalance();
|
||||
if (balance.available()) {
|
||||
return new ProbeOutcome(true, "余额接口可用 surplus=" + balance.surplus()
|
||||
return ProbeOutcome.alive("余额接口可用 surplus=" + balance.surplus()
|
||||
+ " balance=" + balance.balance());
|
||||
}
|
||||
String message = balance.message() == null ? "" : balance.message();
|
||||
@@ -243,9 +243,9 @@ public class NotificationScanScheduler {
|
||||
log.info("[notification-scan] jikip 探测跳过:{}", message);
|
||||
return null;
|
||||
}
|
||||
return new ProbeOutcome(false, message.isEmpty() ? "余额接口不可用" : message);
|
||||
return ProbeOutcome.down(message.isEmpty() ? "余额接口不可用" : message);
|
||||
} catch (Exception ex) {
|
||||
return new ProbeOutcome(false, rootCauseMessage(ex));
|
||||
return ProbeOutcome.down(rootCauseMessage(ex));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +254,13 @@ public class NotificationScanScheduler {
|
||||
if (outcome == null) {
|
||||
return;
|
||||
}
|
||||
if (outcome.verdict() == ProbeVerdict.UNVERIFIED) {
|
||||
// 服务回了个「证明不了业务可用」的状态码(401/403/404):既不能报正常,
|
||||
// 也不能报不可用——报出去就是把探活脚本的问题写成下游事故。
|
||||
log.warn("[notification-scan] 服务探测未验证业务可用性 service={} detail={}"
|
||||
+ "(不发服务不可用通知,请核对探测地址与鉴权)", serviceLabel, outcome.detail());
|
||||
return;
|
||||
}
|
||||
if (outcome.ok()) {
|
||||
log.info("[notification-scan] 服务探测正常 service={} detail={}", serviceLabel, outcome.detail());
|
||||
return;
|
||||
@@ -349,7 +356,45 @@ public class NotificationScanScheduler {
|
||||
LocalDateTime updatedAt) {
|
||||
}
|
||||
|
||||
/** 探测三态:ALIVE=业务可用;UNVERIFIED=服务回了话但证不了业务可用;DOWN=故障。 */
|
||||
enum ProbeVerdict { ALIVE, UNVERIFIED, DOWN }
|
||||
|
||||
/**
|
||||
* 状态码归类。
|
||||
*
|
||||
* 旧口径是「任意 HTTP < 500 视为存活」,于是 404(探测路径没人接)、401/403
|
||||
* (探活侧没带鉴权)都被写成「服务探测正常」——现网品牌检测服务就是这样绿了三个小时。
|
||||
* 这三个码只说明「对端有 HTTP 服务」,不说明被测业务可用,所以单独一态:不报正常、
|
||||
* 也不推「服务不可用」(推出去就是把探活脚本的问题写成下游事故)。其余 4xx 与 5xx 仍算故障。
|
||||
*/
|
||||
static ProbeOutcome probeOutcomeFor(int status, long latencyMillis) {
|
||||
String detail = "HTTP " + status + "(" + latencyMillis + "ms)";
|
||||
if (status >= 200 && status < 400) {
|
||||
return ProbeOutcome.alive(detail);
|
||||
}
|
||||
if (status == 401 || status == 403 || status == 404) {
|
||||
return ProbeOutcome.unverified(detail + ",未验证业务可用性");
|
||||
}
|
||||
return ProbeOutcome.down(detail);
|
||||
}
|
||||
|
||||
/** 探测结果。 */
|
||||
record ProbeOutcome(boolean ok, String detail) {
|
||||
record ProbeOutcome(ProbeVerdict verdict, String detail) {
|
||||
|
||||
static ProbeOutcome alive(String detail) {
|
||||
return new ProbeOutcome(ProbeVerdict.ALIVE, detail);
|
||||
}
|
||||
|
||||
static ProbeOutcome unverified(String detail) {
|
||||
return new ProbeOutcome(ProbeVerdict.UNVERIFIED, detail);
|
||||
}
|
||||
|
||||
static ProbeOutcome down(String detail) {
|
||||
return new ProbeOutcome(ProbeVerdict.DOWN, detail);
|
||||
}
|
||||
|
||||
boolean ok() {
|
||||
return verdict == ProbeVerdict.ALIVE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
@@ -36,6 +36,9 @@ public class NotificationService {
|
||||
public static final String SCENE_SECRET_BALANCE = "secret_balance";
|
||||
public static final String SCENE_SECRET_INVALID = "secret_invalid";
|
||||
public static final String SCENE_TASK_FAILED = "task_failed";
|
||||
/** 任务本身成功、但有一部分行没跑出来(含其自动补测跟进)。大类与 task_failed 同为「任务异常」,
|
||||
* scene 拆开是为了让「交付不完整」不再被按「任务失败」计数。 */
|
||||
public static final String SCENE_TASK_PARTIAL = "task_partial";
|
||||
public static final String SCENE_SERVICE_DOWN = "service_down";
|
||||
/** 麦象(18960 任务调度)异常:任务停滞/失败/队列积压。 */
|
||||
public static final String SCENE_MAIXIANG_ANOMALY = "maixiang_anomaly";
|
||||
@@ -58,6 +61,7 @@ public class NotificationService {
|
||||
SCENE_SERVICE_DOWN, CATEGORY_SYSTEM_ERROR,
|
||||
SCENE_MAIXIANG_ANOMALY, CATEGORY_SYSTEM_ERROR,
|
||||
SCENE_TASK_FAILED, CATEGORY_TASK_ERROR,
|
||||
SCENE_TASK_PARTIAL, CATEGORY_TASK_ERROR,
|
||||
SCENE_SECRET_BALANCE, CATEGORY_CONFIG_ERROR,
|
||||
SCENE_SECRET_INVALID, CATEGORY_CONFIG_ERROR,
|
||||
SCENE_SYSTEM, CATEGORY_SYSTEM_NOTICE);
|
||||
|
||||
+23
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.notification.client;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.NotificationProperties;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.ConsoleReadout;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.WorkerSnapshot;
|
||||
@@ -142,6 +143,28 @@ class MaixiangConsoleClientTest {
|
||||
assertThat(client.parseEnvelope("not-json", "test")).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyReadoutNamesEveryUnreadableShape() {
|
||||
// 这四种形态原先都被折叠成「空数据」,扫描器看不出自己已经瞎了
|
||||
assertThat(client.classifyReadout(null, "Connection refused: connect").readable()).isFalse();
|
||||
assertThat(client.classifyReadout(null, "Connection refused: connect").reason())
|
||||
.contains("Connection refused");
|
||||
assertThat(client.classifyReadout("", null).readable()).isFalse();
|
||||
assertThat(client.classifyReadout("", null).reason()).contains("空");
|
||||
assertThat(client.classifyReadout("<h1>502 Bad Gateway</h1>", null).readable()).isFalse();
|
||||
assertThat(client.classifyReadout("<h1>502 Bad Gateway</h1>", null).reason()).contains("JSON");
|
||||
// 令牌失效 / 接口未部署 / 契约变更:HTTP 200 但 code!=1,mes 必须进告警文案
|
||||
ConsoleReadout rejected = client.classifyReadout("{\"code\":-1,\"mes\":\"令牌无效\",\"data\":null}", null);
|
||||
assertThat(rejected.readable()).isFalse();
|
||||
assertThat(rejected.reason()).contains("令牌无效");
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyReadoutAcceptsSuccessEnvelope() {
|
||||
assertThat(client.classifyReadout("{\"code\":1,\"mes\":\"成功\",\"data\":{\"queues\":[]}}", null).readable())
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseToleratesNullUidAndBadTime() {
|
||||
BatchTaskPage page = client.parseBatchTaskPage(client.parseEnvelope(
|
||||
|
||||
+44
@@ -5,6 +5,7 @@ import com.nanri.aiimage.config.NotificationProperties;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.BatchTaskPage;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.ConsoleReadout;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.QueueStateItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskItem;
|
||||
import com.nanri.aiimage.modules.notification.client.MaixiangConsoleClient.SingleTaskPage;
|
||||
@@ -72,6 +73,48 @@ class MaixiangAnomalyScannerTest {
|
||||
verify(consoleClient, never()).queueStatus();
|
||||
}
|
||||
|
||||
// ======================== 后台可读性 ========================
|
||||
|
||||
@Test
|
||||
void scanAlertsOnceAndSkipsChecksWhenConsoleUnreadable() {
|
||||
// 六项检查全靠后台接口取数:读不到时它们只会「查无异常」,整轮扫描静默失效
|
||||
when(consoleClient.configured()).thenReturn(true);
|
||||
when(dispatch.prepareAdminAudience()).thenReturn(audience);
|
||||
when(consoleClient.probeReadable())
|
||||
.thenReturn(ConsoleReadout.unreadable("接口返回失败:token 无效"));
|
||||
|
||||
newScanner().scan();
|
||||
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象后台数据不可读"),
|
||||
content.capture(), key.capture(), isNull());
|
||||
assertThat(content.getValue()).contains("token 无效");
|
||||
// scan() 自己取当前时间(NOW 常量注入不进来),这里钉住「按自然日去重」的形状而不是某一天
|
||||
assertThat(key.getValue()).matches("maixiang_console_unreadable:\\d{8}");
|
||||
verify(consoleClient, never()).batchTasks(anyInt(), any(), anyInt());
|
||||
verify(consoleClient, never()).singleTasks(anyInt(), any(), anyInt());
|
||||
verify(consoleClient, never()).queueStatus();
|
||||
verify(consoleClient, never()).workers();
|
||||
}
|
||||
|
||||
@Test
|
||||
void scanRunsChecksWhenConsoleReadable() {
|
||||
when(consoleClient.configured()).thenReturn(true);
|
||||
when(dispatch.prepareAdminAudience()).thenReturn(audience);
|
||||
when(consoleClient.probeReadable()).thenReturn(ConsoleReadout.OK);
|
||||
when(consoleClient.workers()).thenReturn(WorkerSnapshot.EMPTY);
|
||||
when(consoleClient.queueStatus()).thenReturn(List.of());
|
||||
when(consoleClient.batchTasks(anyInt(), any(), anyInt())).thenReturn(BatchTaskPage.EMPTY);
|
||||
when(consoleClient.singleTasks(anyInt(), any(), anyInt())).thenReturn(SingleTaskPage.EMPTY);
|
||||
|
||||
newScanner().scan();
|
||||
|
||||
verify(consoleClient, times(1)).queueStatus();
|
||||
verify(dispatch, never()).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
// ======================== 批量任务停滞 ========================
|
||||
|
||||
@Test
|
||||
@@ -373,6 +416,7 @@ class MaixiangAnomalyScannerTest {
|
||||
properties.setMaixiangConsoleToken("token");
|
||||
when(consoleClient.configured()).thenReturn(true);
|
||||
when(dispatch.prepareAdminAudience()).thenReturn(audience);
|
||||
when(consoleClient.probeReadable()).thenReturn(ConsoleReadout.OK);
|
||||
// 仅批量任务停滞检查(status=0 拉取)抛异常
|
||||
doThrow(new RuntimeException("麦象接口超时")).when(consoleClient).batchTasks(eq(0), any(), anyInt());
|
||||
doReturn(BatchTaskPage.EMPTY).when(consoleClient).batchTasks(eq(1), any(), anyInt());
|
||||
|
||||
+3
-3
@@ -40,7 +40,7 @@ class NotificationDispatchServiceRecheckTest {
|
||||
void pushRecheckOutcomeNotifiesUserWithInfoLevelWhenAllResolved() {
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_partial"),
|
||||
eq("info"), contains("补测完成"), contains("新增结果 3 行"), startsWith("collect_recheck:29842:"));
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ class NotificationDispatchServiceRecheckTest {
|
||||
void pushRecheckOutcomeUsesWarningWhileRowsStillPending() {
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 0, 5, 0);
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_partial"),
|
||||
eq("warning"), contains("补测完成"), contains("仍有 5 行"), anyString());
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ class NotificationDispatchServiceRecheckTest {
|
||||
void pushRecheckOutcomeAdvisesManualRerunWhenAllExhausted() {
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 0, 4);
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_partial"),
|
||||
eq("warning"), contains("未能完成"), contains("共 4 行"), startsWith("collect_recheck:29842:"));
|
||||
}
|
||||
|
||||
|
||||
+24
@@ -12,6 +12,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -159,6 +160,29 @@ class NotificationDispatchServiceTest {
|
||||
assertThat(pushed).isEqualTo(3);
|
||||
}
|
||||
|
||||
/**
|
||||
* 「部分结果缺失」必须落在自己的 scene:它和终态失败共用 task_failed 时,巡检按
|
||||
* task_failed 独立桶数报「窗口内 N 条独立失败」会把「任务成功、少跑几行」也算成任务失败
|
||||
* (2026-10-05 值班:窗口 22 行里 20 行是这种)。去重键与大类都不变,只拆 scene。
|
||||
*/
|
||||
@Test
|
||||
void pushPartialResultUsesTaskPartialScene() {
|
||||
AdminUserEntity superAdmin = admin(1L, "超管甲");
|
||||
when(adminUserMapper.selectList(any())).thenReturn(List.of(superAdmin));
|
||||
when(adminAuthSupport.currentRole(superAdmin)).thenReturn("super_admin");
|
||||
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
.thenReturn(true);
|
||||
|
||||
service.pushPartialResult(20L, 3301L, "BRAND", "BRAND-3301", "12 个品牌查询失败");
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(20L), eq(NotificationService.AUDIENCE_USER),
|
||||
eq(NotificationService.SCENE_TASK_PARTIAL), eq(NotificationService.LEVEL_WARNING),
|
||||
contains("任务部分结果缺失"), contains("12 个品牌查询失败"), eq("task_partial:3301"));
|
||||
verify(notificationService).pushOrRefresh(eq(1L), eq(NotificationService.AUDIENCE_ADMIN),
|
||||
eq(NotificationService.SCENE_TASK_PARTIAL), eq(NotificationService.LEVEL_WARNING),
|
||||
contains("用户"), anyString(), eq("task_partial_admin:3301:1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pushToUserUsesUserAudience() {
|
||||
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
|
||||
+95
@@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -184,6 +185,100 @@ class NotificationScanSchedulerTest {
|
||||
assertThat(keyBase.getValue()).matches("service_down:brand-service:\\d{8}");
|
||||
}
|
||||
|
||||
/** 状态码三态口径:4xx 里只有 401/403/404 是「探活不可信」,其余按故障算。 */
|
||||
@Test
|
||||
void httpStatusClassifiesIntoThreeProbeVerdicts() {
|
||||
assertThat(NotificationScanScheduler.probeOutcomeFor(200, 5).verdict())
|
||||
.isEqualTo(NotificationScanScheduler.ProbeVerdict.ALIVE);
|
||||
assertThat(NotificationScanScheduler.probeOutcomeFor(302, 5).verdict())
|
||||
.isEqualTo(NotificationScanScheduler.ProbeVerdict.ALIVE);
|
||||
|
||||
for (int status : new int[]{401, 403, 404}) {
|
||||
NotificationScanScheduler.ProbeOutcome outcome =
|
||||
NotificationScanScheduler.probeOutcomeFor(status, 5);
|
||||
assertThat(outcome.verdict())
|
||||
.as("HTTP %d 不能再被当成「服务存活」:路径没人接/鉴权失败都证不了业务可用", status)
|
||||
.isEqualTo(NotificationScanScheduler.ProbeVerdict.UNVERIFIED);
|
||||
assertThat(outcome.detail()).contains("HTTP " + status).contains("未验证");
|
||||
assertThat(outcome.ok()).isFalse();
|
||||
}
|
||||
|
||||
for (int status : new int[]{400, 405, 429, 500, 502, 503}) {
|
||||
assertThat(NotificationScanScheduler.probeOutcomeFor(status, 5).verdict())
|
||||
.as("HTTP %d 是故障", status)
|
||||
.isEqualTo(NotificationScanScheduler.ProbeVerdict.DOWN);
|
||||
}
|
||||
}
|
||||
|
||||
/** 真发一次 HTTP:404 判「探活不可信」,并且不重试(确定性结果打两次只多一份日志)。 */
|
||||
@Test
|
||||
void probeAgainstHttp404IsUnverifiedAndNotRetried() throws Exception {
|
||||
java.util.concurrent.atomic.AtomicInteger hits = new java.util.concurrent.atomic.AtomicInteger();
|
||||
com.sun.net.httpserver.HttpServer server = startFixedStatusServer(404, hits);
|
||||
NotificationScanScheduler.ProbeOutcome outcome;
|
||||
try {
|
||||
outcome = newScheduler().probeHttpWithRetry(
|
||||
"http://127.0.0.1:" + server.getAddress().getPort() + "/api/version");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
|
||||
assertThat(outcome.verdict()).isEqualTo(NotificationScanScheduler.ProbeVerdict.UNVERIFIED);
|
||||
assertThat(outcome.detail()).contains("未验证");
|
||||
assertThat(hits.get()).isEqualTo(1);
|
||||
}
|
||||
|
||||
/** 反向对照:真故障仍然「失败立即重试一次,两次都失败才算故障」,别把瞬时抖动报出去。 */
|
||||
@Test
|
||||
void probeOnHttp503RetriesOnceThenDeclaresOutage() throws Exception {
|
||||
java.util.concurrent.atomic.AtomicInteger hits = new java.util.concurrent.atomic.AtomicInteger();
|
||||
com.sun.net.httpserver.HttpServer server = startFixedStatusServer(503, hits);
|
||||
NotificationScanScheduler.ProbeOutcome outcome;
|
||||
try {
|
||||
outcome = newScheduler().probeHttpWithRetry(
|
||||
"http://127.0.0.1:" + server.getAddress().getPort() + "/api/version");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
|
||||
assertThat(outcome.verdict()).isEqualTo(NotificationScanScheduler.ProbeVerdict.DOWN);
|
||||
assertThat(hits.get()).isEqualTo(2);
|
||||
}
|
||||
|
||||
/** 404 不推「服务不可用」通知——三态改造不许顺手把它升级成误报(现网最大一类误报源)。 */
|
||||
@Test
|
||||
void probeOnHttp404DoesNotPushServiceDown() throws Exception {
|
||||
com.sun.net.httpserver.HttpServer server = startFixedStatusServer(404,
|
||||
new java.util.concurrent.atomic.AtomicInteger());
|
||||
try {
|
||||
properties.setBrandServiceUrl("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
properties.setPriceTrackApiUrl("");
|
||||
properties.setJikipProbeEnabled(false);
|
||||
|
||||
newScheduler().probeServices();
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
|
||||
verify(dispatch, never()).pushToAdmins(anyString(), anyString(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
private com.sun.net.httpserver.HttpServer startFixedStatusServer(
|
||||
int status, java.util.concurrent.atomic.AtomicInteger hits) throws java.io.IOException {
|
||||
com.sun.net.httpserver.HttpServer server =
|
||||
com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/", exchange -> {
|
||||
hits.incrementAndGet();
|
||||
byte[] body = "probe".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(status, body.length);
|
||||
exchange.getResponseBody().write(body);
|
||||
exchange.close();
|
||||
});
|
||||
server.setExecutor(null);
|
||||
server.start();
|
||||
return server;
|
||||
}
|
||||
|
||||
private void stubAdminAudience() {
|
||||
when(dispatch.prepareAdminAudience()).thenReturn(
|
||||
new NotificationDispatchService.AdminAudience(List.of(), Set.of(), Map.of()));
|
||||
|
||||
+16
@@ -260,6 +260,22 @@ class NotificationServiceTest {
|
||||
.isEqualTo(NotificationService.CATEGORY_SYSTEM_NOTICE);
|
||||
}
|
||||
|
||||
/**
|
||||
* task_partial(交付不完整)与 task_failed(终态失败)是两个 scene,但用户可见大类都必须是
|
||||
* 「任务异常」——拆开 scene 是为了让巡检/看板不再把「少跑了几行」数成「任务失败」,
|
||||
* 不是为了改铃铛的分类。漏登记会兜底进「系统通知」,用户按「任务异常」筛就凭空少一批。
|
||||
*/
|
||||
@Test
|
||||
void taskPartialAndTaskFailedAreSeparateScenesInSameCategory() {
|
||||
assertThat(NotificationService.SCENE_TASK_PARTIAL).isNotEqualTo(NotificationService.SCENE_TASK_FAILED);
|
||||
assertThat(NotificationService.categoryOf(NotificationService.SCENE_TASK_PARTIAL))
|
||||
.isEqualTo(NotificationService.CATEGORY_TASK_ERROR);
|
||||
assertThat(NotificationService.categoryOf(NotificationService.SCENE_TASK_FAILED))
|
||||
.isEqualTo(NotificationService.CATEGORY_TASK_ERROR);
|
||||
assertThat(NotificationService.scenesOfCategory(NotificationService.CATEGORY_TASK_ERROR))
|
||||
.contains(NotificationService.SCENE_TASK_FAILED, NotificationService.SCENE_TASK_PARTIAL);
|
||||
}
|
||||
|
||||
/** 类型筛选落到 SQL 上是 scene IN (...);空/未知分类不追加条件,避免前端传错值就查不到数据。 */
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
Reference in New Issue
Block a user