feat(MQ降级): 补可巡检的计数出口——task.file.job.mq.degraded 指标 + 8 例纯单元
§2 那行「保留回落本地 worker 的语义,但必须产出可巡检的计数出口,不许只有日志」的后半句 从没落地:三条降级出口全是 log.warn + return false,通知表与看板都不感知。 - MeterRegistry 走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径), 降级判定与 return false 语义一字未动 ⇒ 现网零行为变更;不新增 catch(CatchSilenceRatchet 钉着四数,加一个 catch 就判红) - reason 两支:template-missing(模板取不到)、send-failed(熔断打开 / 发送失败各计一次) - 8 例含一条反向钉"正常发送时该指标 0 次 increment",防"无脑 increment"蒙绿; 变异自证:摘掉 template-missing 那处 increment ⇒ 2 例红(expected 1.0),还原后字节哈希相等并复跑绿
This commit is contained in:
+38
-2
@@ -1,10 +1,12 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.rocketmq.spring.core.RocketMQTemplate;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -13,10 +15,15 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskFileJobPublisher {
|
||||
|
||||
/** 降级计数出口:MQ 回落本地 worker 的每条出口都要被数得到,不再只有日志。 */
|
||||
static final String DEGRADED_METRIC = "task.file.job.mq.degraded";
|
||||
static final String REASON_TEMPLATE_MISSING = "template-missing";
|
||||
static final String REASON_SEND_FAILED = "send-failed";
|
||||
|
||||
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
|
||||
private final MeterRegistry meterRegistry; // 可为 null:没配 Micrometer 注册表时只留日志,降级判定与回落语义一字不动
|
||||
private final AtomicInteger consecutiveFailures = new AtomicInteger();
|
||||
private final AtomicLong circuitOpenUntilMillis = new AtomicLong();
|
||||
|
||||
@@ -35,6 +42,22 @@ public class TaskFileJobPublisher {
|
||||
@Value("${aiimage.result-file-job.mq-circuit-open-ms:60000}")
|
||||
private long mqCircuitOpenMillis;
|
||||
|
||||
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider) {
|
||||
this(rocketMQTemplateProvider, (MeterRegistry) null);
|
||||
}
|
||||
|
||||
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider, MeterRegistry meterRegistry) {
|
||||
this.rocketMQTemplateProvider = rocketMQTemplateProvider;
|
||||
this.meterRegistry = meterRegistry;
|
||||
}
|
||||
|
||||
/** 生产构造:注册表走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径)。 */
|
||||
@Autowired
|
||||
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||
this(rocketMQTemplateProvider, meterRegistryProvider == null ? null : meterRegistryProvider.getIfAvailable());
|
||||
}
|
||||
|
||||
public boolean publish(TaskFileJobMessage message) {
|
||||
if (message == null || message.getJobId() == null) {
|
||||
return false;
|
||||
@@ -53,6 +76,7 @@ public class TaskFileJobPublisher {
|
||||
}
|
||||
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
|
||||
if (template == null) {
|
||||
countDegraded(REASON_TEMPLATE_MISSING);
|
||||
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType());
|
||||
return false;
|
||||
@@ -70,14 +94,26 @@ public class TaskFileJobPublisher {
|
||||
if (failures >= threshold) {
|
||||
long nextOpenUntil = System.currentTimeMillis() + Math.max(1000L, mqCircuitOpenMillis);
|
||||
circuitOpenUntilMillis.set(nextOpenUntil);
|
||||
countDegraded(REASON_SEND_FAILED);
|
||||
log.warn("[task-file-job] publish RocketMQ failed, circuit opened and fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={} openMs={} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), failures,
|
||||
Math.max(1000L, mqCircuitOpenMillis), ex.getMessage());
|
||||
} else {
|
||||
countDegraded(REASON_SEND_FAILED);
|
||||
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={}/{} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), failures, threshold, ex.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void countDegraded(String reason) {
|
||||
if (meterRegistry == null) {
|
||||
return;
|
||||
}
|
||||
Counter.builder(DEGRADED_METRIC)
|
||||
.tag("reason", reason)
|
||||
.register(meterRegistry)
|
||||
.increment();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user