feat(MQ降级): 补可巡检的计数出口——task.file.job.mq.degraded 指标 + 8 例纯单元

§2 那行「保留回落本地 worker 的语义,但必须产出可巡检的计数出口,不许只有日志」的后半句
从没落地:三条降级出口全是 log.warn + return false,通知表与看板都不感知。

- MeterRegistry 走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径),
  降级判定与 return false 语义一字未动 ⇒ 现网零行为变更;不新增 catch(CatchSilenceRatchet
  钉着四数,加一个 catch 就判红)
- reason 两支:template-missing(模板取不到)、send-failed(熔断打开 / 发送失败各计一次)
- 8 例含一条反向钉"正常发送时该指标 0 次 increment",防"无脑 increment"蒙绿;
  变异自证:摘掉 template-missing 那处 increment ⇒ 2 例红(expected 1.0),还原后字节哈希相等并复跑绿
This commit is contained in:
2026-10-09 13:12:51 +08:00
parent 32a5dec550
commit b4b2fe2aa5
2 changed files with 240 additions and 2 deletions
@@ -1,10 +1,12 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -13,10 +15,15 @@ import java.util.concurrent.atomic.AtomicLong;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobPublisher {
/** 降级计数出口:MQ 回落本地 worker 的每条出口都要被数得到,不再只有日志。 */
static final String DEGRADED_METRIC = "task.file.job.mq.degraded";
static final String REASON_TEMPLATE_MISSING = "template-missing";
static final String REASON_SEND_FAILED = "send-failed";
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
private final MeterRegistry meterRegistry; // 可为 null:没配 Micrometer 注册表时只留日志,降级判定与回落语义一字不动
private final AtomicInteger consecutiveFailures = new AtomicInteger();
private final AtomicLong circuitOpenUntilMillis = new AtomicLong();
@@ -35,6 +42,22 @@ public class TaskFileJobPublisher {
@Value("${aiimage.result-file-job.mq-circuit-open-ms:60000}")
private long mqCircuitOpenMillis;
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider) {
this(rocketMQTemplateProvider, (MeterRegistry) null);
}
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider, MeterRegistry meterRegistry) {
this.rocketMQTemplateProvider = rocketMQTemplateProvider;
this.meterRegistry = meterRegistry;
}
/** 生产构造:注册表走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径)。 */
@Autowired
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider,
ObjectProvider<MeterRegistry> meterRegistryProvider) {
this(rocketMQTemplateProvider, meterRegistryProvider == null ? null : meterRegistryProvider.getIfAvailable());
}
public boolean publish(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return false;
@@ -53,6 +76,7 @@ public class TaskFileJobPublisher {
}
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
if (template == null) {
countDegraded(REASON_TEMPLATE_MISSING);
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
@@ -70,14 +94,26 @@ public class TaskFileJobPublisher {
if (failures >= threshold) {
long nextOpenUntil = System.currentTimeMillis() + Math.max(1000L, mqCircuitOpenMillis);
circuitOpenUntilMillis.set(nextOpenUntil);
countDegraded(REASON_SEND_FAILED);
log.warn("[task-file-job] publish RocketMQ failed, circuit opened and fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={} openMs={} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), failures,
Math.max(1000L, mqCircuitOpenMillis), ex.getMessage());
} else {
countDegraded(REASON_SEND_FAILED);
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={}/{} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), failures, threshold, ex.getMessage());
}
return false;
}
}
private void countDegraded(String reason) {
if (meterRegistry == null) {
return;
}
Counter.builder(DEGRADED_METRIC)
.tag("reason", reason)
.register(meterRegistry)
.increment();
}
}
@@ -0,0 +1,202 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.concurrent.atomic.AtomicLong;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* RocketMQ 降级路径的**可巡检计数出口**契约:{@code task.file.job.mq.degraded} 按 {@code reason}
* 分两档({@code template-missing} / {@code send-failed}),三处回落本地 worker 的降级点各自计数,
* 而成功发送一条都不许计。
*
* <p>三条判据各挡一种蒙混:只计一档 ⇒ 反向钉(reason 用错)红;把两处 send-failed 合成一处计
* ⇒ 阈值前后累计那条红;无条件无脑计 ⇒ 成功用例的注册表非空那条红。
*
* <p>另一半契约是**现网零行为变更**:没有 MeterRegistry(1 参构造 / ObjectProvider 取不到 / 显式传 null)
* 时必须照旧返回 false、照旧开熔断,绝不因为"没指标"而改判据。
*
* <p>重跑:{@code mvn -o test -Dtest=TaskFileJobPublisherDegradedCounterTest}(cwd = backend-java,纯 mock 不连库)。
*/
class TaskFileJobPublisherDegradedCounterTest {
private static final String METRIC = "task.file.job.mq.degraded";
private static final String REASON_TEMPLATE_MISSING = "template-missing";
private static final String REASON_SEND_FAILED = "send-failed";
@SuppressWarnings("unchecked")
private static ObjectProvider<RocketMQTemplate> templateProvider(RocketMQTemplate template) {
ObjectProvider<RocketMQTemplate> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(template);
return provider;
}
@SuppressWarnings("unchecked")
private static ObjectProvider<MeterRegistry> registryProvider(MeterRegistry registry) {
ObjectProvider<MeterRegistry> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(registry);
return provider;
}
private static TaskFileJobPublisher configured(TaskFileJobPublisher publisher, int failureThreshold) {
ReflectionTestUtils.setField(publisher, "mqEnabled", true);
ReflectionTestUtils.setField(publisher, "topic", "aiimage-result-file-job");
ReflectionTestUtils.setField(publisher, "mqSendTimeoutMillis", 1000L);
ReflectionTestUtils.setField(publisher, "mqFailureThreshold", failureThreshold);
ReflectionTestUtils.setField(publisher, "mqCircuitOpenMillis", 60000L);
return publisher;
}
private static TaskFileJobPublisher publisher(MeterRegistry registry, RocketMQTemplate template,
int failureThreshold) {
return configured(new TaskFileJobPublisher(templateProvider(template), registry), failureThreshold);
}
private static RocketMQTemplate failingTemplate() {
RocketMQTemplate template = mock(RocketMQTemplate.class);
when(template.syncSend(anyString(), any(Object.class), anyLong()))
.thenThrow(new RuntimeException("broker down"));
return template;
}
private static TaskFileJobMessage message(long jobId) {
TaskFileJobMessage message = new TaskFileJobMessage();
message.setJobId(jobId);
message.setTaskId(900L + jobId);
message.setModuleType("similarasin");
return message;
}
private static double count(MeterRegistry registry, String reason) {
Counter counter = registry.find(METRIC).tag("reason", reason).counter();
return counter == null ? 0.0d : counter.count();
}
private static long circuitOpenUntilMillis(TaskFileJobPublisher publisher) {
AtomicLong openUntil = (AtomicLong) ReflectionTestUtils.getField(publisher, "circuitOpenUntilMillis");
return openUntil == null ? 0L : openUntil.get();
}
// -------------------------------------------------- 降级点 1:RocketMQTemplate 取不到
@Test
void missingTemplateCountsDegradedOnceUnderTemplateMissingReason() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
TaskFileJobPublisher publisher = publisher(registry, null, 3);
assertThat(publisher.publish(message(1L))).as("取不到 template 仍回落本地 worker").isFalse();
assertThat(count(registry, REASON_TEMPLATE_MISSING)).isEqualTo(1.0d);
assertThat(count(registry, REASON_SEND_FAILED)).as("这条降级不是发送失败,reason 不许串档").isZero();
}
// -------------------------------------------------- 降级点 2:发送失败但未达阈值
@Test
void sendFailureBelowThresholdCountsDegradedUnderSendFailedReason() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 3);
assertThat(publisher.publish(message(2L))).as("发送失败仍回落本地 worker").isFalse();
assertThat(count(registry, REASON_SEND_FAILED)).isEqualTo(1.0d);
assertThat(count(registry, REASON_TEMPLATE_MISSING)).isZero();
assertThat(circuitOpenUntilMillis(publisher)).as("未达阈值不许提前开熔断").isZero();
}
// -------------------------------------------------- 降级点 3:发送失败并把熔断打开
@Test
void sendFailureThatOpensTheCircuitAlsoCountsDegradedUnderSendFailedReason() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 1);
assertThat(publisher.publish(message(3L))).isFalse();
assertThat(count(registry, REASON_SEND_FAILED)).isEqualTo(1.0d);
assertThat(circuitOpenUntilMillis(publisher)).as("熔断打开那条也得被数得到")
.isGreaterThan(System.currentTimeMillis());
}
/**
* 两处 send-failed 分支**各自** increment:同一个 publisher 先走"未达阈值"再走"熔断打开",
* 计数必须累到 2。把两条合成一条计(或只在 catch 入口计一次)在这里红。
*/
@Test
void bothSendFailureBranchesEachIncrementTheSameReason() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 2);
assertThat(publisher.publish(message(4L))).isFalse();
assertThat(count(registry, REASON_SEND_FAILED)).as("未达阈值那条应计 1").isEqualTo(1.0d);
assertThat(circuitOpenUntilMillis(publisher)).isZero();
assertThat(publisher.publish(message(4L))).isFalse();
assertThat(count(registry, REASON_SEND_FAILED)).as("熔断打开那条应再计 1,累计 2").isEqualTo(2.0d);
assertThat(circuitOpenUntilMillis(publisher)).isGreaterThan(System.currentTimeMillis());
assertThat(registry.find(METRIC).tag("reason", REASON_TEMPLATE_MISSING).counter())
.as("两处都归 send-failed,不许冒出第三个 reason").isNull();
}
// -------------------------------------------------- 反向钉:成功路径一次都不许多计
@Test
void successfulPublishIncrementsNothing() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
TaskFileJobPublisher publisher = publisher(registry, mock(RocketMQTemplate.class), 3);
assertThat(publisher.publish(message(5L))).as("发送成功应回 true(语义未变)").isTrue();
assertThat(registry.getMeters()).as("成功路径不许注册任何降级指标").isEmpty();
assertThat(registry.find(METRIC).counter()).isNull();
assertThat(circuitOpenUntilMillis(publisher)).isZero();
}
// -------------------------------------------------- 零行为变更:没有注册表也要照常回落
@Test
void nullMeterRegistryKeepsTheDegradedSemanticsAndStillFallsBack() {
TaskFileJobPublisher withoutRegistry = publisher(null, null, 3);
assertThat(withoutRegistry.publish(message(6L))).as("无注册表时降级判定不变").isFalse();
TaskFileJobPublisher legacyOneArgCtor =
configured(new TaskFileJobPublisher(templateProvider(failingTemplate())), 1);
assertThat(legacyOneArgCtor.publish(message(7L))).as("1 参构造(旧签名)行为不变").isFalse();
assertThat(circuitOpenUntilMillis(legacyOneArgCtor)).as("无注册表也要照常开熔断")
.isGreaterThan(System.currentTimeMillis());
}
@Test
void degradedCounterIsTakenFromTheObjectProviderChannel() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
TaskFileJobPublisher publisher = configured(
new TaskFileJobPublisher(templateProvider(null), registryProvider(registry)), 3);
assertThat(publisher.publish(message(8L))).isFalse();
assertThat(count(registry, REASON_TEMPLATE_MISSING)).as("生产构造要走 ObjectProvider 取注册表")
.isEqualTo(1.0d);
}
@Test
void emptyObjectProviderFallsBackToNoRegistryWithoutChangingBehavior() {
TaskFileJobPublisher publisher = configured(
new TaskFileJobPublisher(templateProvider(failingTemplate()), registryProvider(null)), 3);
assertThat(publisher.publish(message(9L))).isFalse();
assertThat(circuitOpenUntilMillis(publisher)).as("取不到注册表时熔断照旧").isZero();
}
}