feat(MQ降级): 补可巡检的计数出口——task.file.job.mq.degraded 指标 + 8 例纯单元

§2 那行「保留回落本地 worker 的语义,但必须产出可巡检的计数出口,不许只有日志」的后半句
从没落地:三条降级出口全是 log.warn + return false,通知表与看板都不感知。

- MeterRegistry 走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径),
  降级判定与 return false 语义一字未动 ⇒ 现网零行为变更;不新增 catch(CatchSilenceRatchet
  钉着四数,加一个 catch 就判红)
- reason 两支:template-missing(模板取不到)、send-failed(熔断打开 / 发送失败各计一次)
- 8 例含一条反向钉"正常发送时该指标 0 次 increment",防"无脑 increment"蒙绿;
  变异自证:摘掉 template-missing 那处 increment ⇒ 2 例红(expected 1.0),还原后字节哈希相等并复跑绿
This commit is contained in:
2026-10-09 13:12:51 +08:00
parent 32a5dec550
commit b4b2fe2aa5
2 changed files with 240 additions and 2 deletions
@@ -1,10 +1,12 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -13,10 +15,15 @@ import java.util.concurrent.atomic.AtomicLong;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobPublisher {
/** 降级计数出口:MQ 回落本地 worker 的每条出口都要被数得到,不再只有日志。 */
static final String DEGRADED_METRIC = "task.file.job.mq.degraded";
static final String REASON_TEMPLATE_MISSING = "template-missing";
static final String REASON_SEND_FAILED = "send-failed";
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
private final MeterRegistry meterRegistry; // 可为 null:没配 Micrometer 注册表时只留日志,降级判定与回落语义一字不动
private final AtomicInteger consecutiveFailures = new AtomicInteger();
private final AtomicLong circuitOpenUntilMillis = new AtomicLong();
@@ -35,6 +42,22 @@ public class TaskFileJobPublisher {
@Value("${aiimage.result-file-job.mq-circuit-open-ms:60000}")
private long mqCircuitOpenMillis;
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider) {
this(rocketMQTemplateProvider, (MeterRegistry) null);
}
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider, MeterRegistry meterRegistry) {
this.rocketMQTemplateProvider = rocketMQTemplateProvider;
this.meterRegistry = meterRegistry;
}
/** 生产构造:注册表走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径)。 */
@Autowired
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider,
ObjectProvider<MeterRegistry> meterRegistryProvider) {
this(rocketMQTemplateProvider, meterRegistryProvider == null ? null : meterRegistryProvider.getIfAvailable());
}
public boolean publish(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return false;
@@ -53,6 +76,7 @@ public class TaskFileJobPublisher {
}
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
if (template == null) {
countDegraded(REASON_TEMPLATE_MISSING);
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
@@ -70,14 +94,26 @@ public class TaskFileJobPublisher {
if (failures >= threshold) {
long nextOpenUntil = System.currentTimeMillis() + Math.max(1000L, mqCircuitOpenMillis);
circuitOpenUntilMillis.set(nextOpenUntil);
countDegraded(REASON_SEND_FAILED);
log.warn("[task-file-job] publish RocketMQ failed, circuit opened and fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={} openMs={} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), failures,
Math.max(1000L, mqCircuitOpenMillis), ex.getMessage());
} else {
countDegraded(REASON_SEND_FAILED);
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={}/{} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), failures, threshold, ex.getMessage());
}
return false;
}
}
private void countDegraded(String reason) {
if (meterRegistry == null) {
return;
}
Counter.builder(DEGRADED_METRIC)
.tag("reason", reason)
.register(meterRegistry)
.increment();
}
}