feat(MQ降级): 补可巡检的计数出口——task.file.job.mq.degraded 指标 + 8 例纯单元
§2 那行「保留回落本地 worker 的语义,但必须产出可巡检的计数出口,不许只有日志」的后半句 从没落地:三条降级出口全是 log.warn + return false,通知表与看板都不感知。 - MeterRegistry 走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径), 降级判定与 return false 语义一字未动 ⇒ 现网零行为变更;不新增 catch(CatchSilenceRatchet 钉着四数,加一个 catch 就判红) - reason 两支:template-missing(模板取不到)、send-failed(熔断打开 / 发送失败各计一次) - 8 例含一条反向钉"正常发送时该指标 0 次 increment",防"无脑 increment"蒙绿; 变异自证:摘掉 template-missing 那处 increment ⇒ 2 例红(expected 1.0),还原后字节哈希相等并复跑绿
This commit is contained in:
+38
-2
@@ -1,10 +1,12 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.rocketmq.spring.core.RocketMQTemplate;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -13,10 +15,15 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskFileJobPublisher {
|
||||
|
||||
/** 降级计数出口:MQ 回落本地 worker 的每条出口都要被数得到,不再只有日志。 */
|
||||
static final String DEGRADED_METRIC = "task.file.job.mq.degraded";
|
||||
static final String REASON_TEMPLATE_MISSING = "template-missing";
|
||||
static final String REASON_SEND_FAILED = "send-failed";
|
||||
|
||||
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
|
||||
private final MeterRegistry meterRegistry; // 可为 null:没配 Micrometer 注册表时只留日志,降级判定与回落语义一字不动
|
||||
private final AtomicInteger consecutiveFailures = new AtomicInteger();
|
||||
private final AtomicLong circuitOpenUntilMillis = new AtomicLong();
|
||||
|
||||
@@ -35,6 +42,22 @@ public class TaskFileJobPublisher {
|
||||
@Value("${aiimage.result-file-job.mq-circuit-open-ms:60000}")
|
||||
private long mqCircuitOpenMillis;
|
||||
|
||||
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider) {
|
||||
this(rocketMQTemplateProvider, (MeterRegistry) null);
|
||||
}
|
||||
|
||||
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider, MeterRegistry meterRegistry) {
|
||||
this.rocketMQTemplateProvider = rocketMQTemplateProvider;
|
||||
this.meterRegistry = meterRegistry;
|
||||
}
|
||||
|
||||
/** 生产构造:注册表走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径)。 */
|
||||
@Autowired
|
||||
public TaskFileJobPublisher(ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||
this(rocketMQTemplateProvider, meterRegistryProvider == null ? null : meterRegistryProvider.getIfAvailable());
|
||||
}
|
||||
|
||||
public boolean publish(TaskFileJobMessage message) {
|
||||
if (message == null || message.getJobId() == null) {
|
||||
return false;
|
||||
@@ -53,6 +76,7 @@ public class TaskFileJobPublisher {
|
||||
}
|
||||
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
|
||||
if (template == null) {
|
||||
countDegraded(REASON_TEMPLATE_MISSING);
|
||||
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType());
|
||||
return false;
|
||||
@@ -70,14 +94,26 @@ public class TaskFileJobPublisher {
|
||||
if (failures >= threshold) {
|
||||
long nextOpenUntil = System.currentTimeMillis() + Math.max(1000L, mqCircuitOpenMillis);
|
||||
circuitOpenUntilMillis.set(nextOpenUntil);
|
||||
countDegraded(REASON_SEND_FAILED);
|
||||
log.warn("[task-file-job] publish RocketMQ failed, circuit opened and fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={} openMs={} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), failures,
|
||||
Math.max(1000L, mqCircuitOpenMillis), ex.getMessage());
|
||||
} else {
|
||||
countDegraded(REASON_SEND_FAILED);
|
||||
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={}/{} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), failures, threshold, ex.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void countDegraded(String reason) {
|
||||
if (meterRegistry == null) {
|
||||
return;
|
||||
}
|
||||
Counter.builder(DEGRADED_METRIC)
|
||||
.tag("reason", reason)
|
||||
.register(meterRegistry)
|
||||
.increment();
|
||||
}
|
||||
}
|
||||
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.apache.rocketmq.spring.core.RocketMQTemplate;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* RocketMQ 降级路径的**可巡检计数出口**契约:{@code task.file.job.mq.degraded} 按 {@code reason}
|
||||
* 分两档({@code template-missing} / {@code send-failed}),三处回落本地 worker 的降级点各自计数,
|
||||
* 而成功发送一条都不许计。
|
||||
*
|
||||
* <p>三条判据各挡一种蒙混:只计一档 ⇒ 反向钉(reason 用错)红;把两处 send-failed 合成一处计
|
||||
* ⇒ 阈值前后累计那条红;无条件无脑计 ⇒ 成功用例的注册表非空那条红。
|
||||
*
|
||||
* <p>另一半契约是**现网零行为变更**:没有 MeterRegistry(1 参构造 / ObjectProvider 取不到 / 显式传 null)
|
||||
* 时必须照旧返回 false、照旧开熔断,绝不因为"没指标"而改判据。
|
||||
*
|
||||
* <p>重跑:{@code mvn -o test -Dtest=TaskFileJobPublisherDegradedCounterTest}(cwd = backend-java,纯 mock 不连库)。
|
||||
*/
|
||||
class TaskFileJobPublisherDegradedCounterTest {
|
||||
|
||||
private static final String METRIC = "task.file.job.mq.degraded";
|
||||
private static final String REASON_TEMPLATE_MISSING = "template-missing";
|
||||
private static final String REASON_SEND_FAILED = "send-failed";
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static ObjectProvider<RocketMQTemplate> templateProvider(RocketMQTemplate template) {
|
||||
ObjectProvider<RocketMQTemplate> provider = mock(ObjectProvider.class);
|
||||
when(provider.getIfAvailable()).thenReturn(template);
|
||||
return provider;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static ObjectProvider<MeterRegistry> registryProvider(MeterRegistry registry) {
|
||||
ObjectProvider<MeterRegistry> provider = mock(ObjectProvider.class);
|
||||
when(provider.getIfAvailable()).thenReturn(registry);
|
||||
return provider;
|
||||
}
|
||||
|
||||
private static TaskFileJobPublisher configured(TaskFileJobPublisher publisher, int failureThreshold) {
|
||||
ReflectionTestUtils.setField(publisher, "mqEnabled", true);
|
||||
ReflectionTestUtils.setField(publisher, "topic", "aiimage-result-file-job");
|
||||
ReflectionTestUtils.setField(publisher, "mqSendTimeoutMillis", 1000L);
|
||||
ReflectionTestUtils.setField(publisher, "mqFailureThreshold", failureThreshold);
|
||||
ReflectionTestUtils.setField(publisher, "mqCircuitOpenMillis", 60000L);
|
||||
return publisher;
|
||||
}
|
||||
|
||||
private static TaskFileJobPublisher publisher(MeterRegistry registry, RocketMQTemplate template,
|
||||
int failureThreshold) {
|
||||
return configured(new TaskFileJobPublisher(templateProvider(template), registry), failureThreshold);
|
||||
}
|
||||
|
||||
private static RocketMQTemplate failingTemplate() {
|
||||
RocketMQTemplate template = mock(RocketMQTemplate.class);
|
||||
when(template.syncSend(anyString(), any(Object.class), anyLong()))
|
||||
.thenThrow(new RuntimeException("broker down"));
|
||||
return template;
|
||||
}
|
||||
|
||||
private static TaskFileJobMessage message(long jobId) {
|
||||
TaskFileJobMessage message = new TaskFileJobMessage();
|
||||
message.setJobId(jobId);
|
||||
message.setTaskId(900L + jobId);
|
||||
message.setModuleType("similarasin");
|
||||
return message;
|
||||
}
|
||||
|
||||
private static double count(MeterRegistry registry, String reason) {
|
||||
Counter counter = registry.find(METRIC).tag("reason", reason).counter();
|
||||
return counter == null ? 0.0d : counter.count();
|
||||
}
|
||||
|
||||
private static long circuitOpenUntilMillis(TaskFileJobPublisher publisher) {
|
||||
AtomicLong openUntil = (AtomicLong) ReflectionTestUtils.getField(publisher, "circuitOpenUntilMillis");
|
||||
return openUntil == null ? 0L : openUntil.get();
|
||||
}
|
||||
|
||||
// -------------------------------------------------- 降级点 1:RocketMQTemplate 取不到
|
||||
|
||||
@Test
|
||||
void missingTemplateCountsDegradedOnceUnderTemplateMissingReason() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
TaskFileJobPublisher publisher = publisher(registry, null, 3);
|
||||
|
||||
assertThat(publisher.publish(message(1L))).as("取不到 template 仍回落本地 worker").isFalse();
|
||||
|
||||
assertThat(count(registry, REASON_TEMPLATE_MISSING)).isEqualTo(1.0d);
|
||||
assertThat(count(registry, REASON_SEND_FAILED)).as("这条降级不是发送失败,reason 不许串档").isZero();
|
||||
}
|
||||
|
||||
// -------------------------------------------------- 降级点 2:发送失败但未达阈值
|
||||
|
||||
@Test
|
||||
void sendFailureBelowThresholdCountsDegradedUnderSendFailedReason() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 3);
|
||||
|
||||
assertThat(publisher.publish(message(2L))).as("发送失败仍回落本地 worker").isFalse();
|
||||
|
||||
assertThat(count(registry, REASON_SEND_FAILED)).isEqualTo(1.0d);
|
||||
assertThat(count(registry, REASON_TEMPLATE_MISSING)).isZero();
|
||||
assertThat(circuitOpenUntilMillis(publisher)).as("未达阈值不许提前开熔断").isZero();
|
||||
}
|
||||
|
||||
// -------------------------------------------------- 降级点 3:发送失败并把熔断打开
|
||||
|
||||
@Test
|
||||
void sendFailureThatOpensTheCircuitAlsoCountsDegradedUnderSendFailedReason() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 1);
|
||||
|
||||
assertThat(publisher.publish(message(3L))).isFalse();
|
||||
|
||||
assertThat(count(registry, REASON_SEND_FAILED)).isEqualTo(1.0d);
|
||||
assertThat(circuitOpenUntilMillis(publisher)).as("熔断打开那条也得被数得到")
|
||||
.isGreaterThan(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/**
|
||||
* 两处 send-failed 分支**各自** increment:同一个 publisher 先走"未达阈值"再走"熔断打开",
|
||||
* 计数必须累到 2。把两条合成一条计(或只在 catch 入口计一次)在这里红。
|
||||
*/
|
||||
@Test
|
||||
void bothSendFailureBranchesEachIncrementTheSameReason() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 2);
|
||||
|
||||
assertThat(publisher.publish(message(4L))).isFalse();
|
||||
assertThat(count(registry, REASON_SEND_FAILED)).as("未达阈值那条应计 1").isEqualTo(1.0d);
|
||||
assertThat(circuitOpenUntilMillis(publisher)).isZero();
|
||||
|
||||
assertThat(publisher.publish(message(4L))).isFalse();
|
||||
assertThat(count(registry, REASON_SEND_FAILED)).as("熔断打开那条应再计 1,累计 2").isEqualTo(2.0d);
|
||||
assertThat(circuitOpenUntilMillis(publisher)).isGreaterThan(System.currentTimeMillis());
|
||||
assertThat(registry.find(METRIC).tag("reason", REASON_TEMPLATE_MISSING).counter())
|
||||
.as("两处都归 send-failed,不许冒出第三个 reason").isNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------- 反向钉:成功路径一次都不许多计
|
||||
|
||||
@Test
|
||||
void successfulPublishIncrementsNothing() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
TaskFileJobPublisher publisher = publisher(registry, mock(RocketMQTemplate.class), 3);
|
||||
|
||||
assertThat(publisher.publish(message(5L))).as("发送成功应回 true(语义未变)").isTrue();
|
||||
|
||||
assertThat(registry.getMeters()).as("成功路径不许注册任何降级指标").isEmpty();
|
||||
assertThat(registry.find(METRIC).counter()).isNull();
|
||||
assertThat(circuitOpenUntilMillis(publisher)).isZero();
|
||||
}
|
||||
|
||||
// -------------------------------------------------- 零行为变更:没有注册表也要照常回落
|
||||
|
||||
@Test
|
||||
void nullMeterRegistryKeepsTheDegradedSemanticsAndStillFallsBack() {
|
||||
TaskFileJobPublisher withoutRegistry = publisher(null, null, 3);
|
||||
assertThat(withoutRegistry.publish(message(6L))).as("无注册表时降级判定不变").isFalse();
|
||||
|
||||
TaskFileJobPublisher legacyOneArgCtor =
|
||||
configured(new TaskFileJobPublisher(templateProvider(failingTemplate())), 1);
|
||||
assertThat(legacyOneArgCtor.publish(message(7L))).as("1 参构造(旧签名)行为不变").isFalse();
|
||||
assertThat(circuitOpenUntilMillis(legacyOneArgCtor)).as("无注册表也要照常开熔断")
|
||||
.isGreaterThan(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
void degradedCounterIsTakenFromTheObjectProviderChannel() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
TaskFileJobPublisher publisher = configured(
|
||||
new TaskFileJobPublisher(templateProvider(null), registryProvider(registry)), 3);
|
||||
|
||||
assertThat(publisher.publish(message(8L))).isFalse();
|
||||
|
||||
assertThat(count(registry, REASON_TEMPLATE_MISSING)).as("生产构造要走 ObjectProvider 取注册表")
|
||||
.isEqualTo(1.0d);
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptyObjectProviderFallsBackToNoRegistryWithoutChangingBehavior() {
|
||||
TaskFileJobPublisher publisher = configured(
|
||||
new TaskFileJobPublisher(templateProvider(failingTemplate()), registryProvider(null)), 3);
|
||||
|
||||
assertThat(publisher.publish(message(9L))).isFalse();
|
||||
assertThat(circuitOpenUntilMillis(publisher)).as("取不到注册表时熔断照旧").isZero();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user