From b4b2fe2aa5f977d95ca290d1ca42fca5d090c3cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Fri, 9 Oct 2026 13:12:51 +0800 Subject: [PATCH] =?UTF-8?q?feat(MQ=E9=99=8D=E7=BA=A7):=20=E8=A1=A5?= =?UTF-8?q?=E5=8F=AF=E5=B7=A1=E6=A3=80=E7=9A=84=E8=AE=A1=E6=95=B0=E5=87=BA?= =?UTF-8?q?=E5=8F=A3=E2=80=94=E2=80=94task.file.job.mq.degraded=20?= =?UTF-8?q?=E6=8C=87=E6=A0=87=20+=208=20=E4=BE=8B=E7=BA=AF=E5=8D=95?= =?UTF-8?q?=E5=85=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2 那行「保留回落本地 worker 的语义,但必须产出可巡检的计数出口,不许只有日志」的后半句 从没落地:三条降级出口全是 log.warn + return false,通知表与看板都不感知。 - MeterRegistry 走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径), 降级判定与 return false 语义一字未动 ⇒ 现网零行为变更;不新增 catch(CatchSilenceRatchet 钉着四数,加一个 catch 就判红) - reason 两支:template-missing(模板取不到)、send-failed(熔断打开 / 发送失败各计一次) - 8 例含一条反向钉"正常发送时该指标 0 次 increment",防"无脑 increment"蒙绿; 变异自证:摘掉 template-missing 那处 increment ⇒ 2 例红(expected 1.0),还原后字节哈希相等并复跑绿 --- .../task/service/TaskFileJobPublisher.java | 40 +++- ...skFileJobPublisherDegradedCounterTest.java | 202 ++++++++++++++++++ 2 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisherDegradedCounterTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java index c38979409..6c78ce040 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java @@ -1,10 +1,12 @@ package com.nanri.aiimage.modules.task.service; import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage; -import lombok.RequiredArgsConstructor; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.rocketmq.spring.core.RocketMQTemplate; import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -13,10 +15,15 @@ import java.util.concurrent.atomic.AtomicLong; @Slf4j @Service -@RequiredArgsConstructor public class TaskFileJobPublisher { + /** 降级计数出口:MQ 回落本地 worker 的每条出口都要被数得到,不再只有日志。 */ + static final String DEGRADED_METRIC = "task.file.job.mq.degraded"; + static final String REASON_TEMPLATE_MISSING = "template-missing"; + static final String REASON_SEND_FAILED = "send-failed"; + private final ObjectProvider rocketMQTemplateProvider; + private final MeterRegistry meterRegistry; // 可为 null:没配 Micrometer 注册表时只留日志,降级判定与回落语义一字不动 private final AtomicInteger consecutiveFailures = new AtomicInteger(); private final AtomicLong circuitOpenUntilMillis = new AtomicLong(); @@ -35,6 +42,22 @@ public class TaskFileJobPublisher { @Value("${aiimage.result-file-job.mq-circuit-open-ms:60000}") private long mqCircuitOpenMillis; + public TaskFileJobPublisher(ObjectProvider rocketMQTemplateProvider) { + this(rocketMQTemplateProvider, (MeterRegistry) null); + } + + public TaskFileJobPublisher(ObjectProvider rocketMQTemplateProvider, MeterRegistry meterRegistry) { + this.rocketMQTemplateProvider = rocketMQTemplateProvider; + this.meterRegistry = meterRegistry; + } + + /** 生产构造:注册表走 ObjectProvider,取不到就按 null 兜住(与 ClientVersionGuardFilter 同口径)。 */ + @Autowired + public TaskFileJobPublisher(ObjectProvider rocketMQTemplateProvider, + ObjectProvider meterRegistryProvider) { + this(rocketMQTemplateProvider, meterRegistryProvider == null ? null : meterRegistryProvider.getIfAvailable()); + } + public boolean publish(TaskFileJobMessage message) { if (message == null || message.getJobId() == null) { return false; @@ -53,6 +76,7 @@ public class TaskFileJobPublisher { } RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable(); if (template == null) { + countDegraded(REASON_TEMPLATE_MISSING); log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}", message.getJobId(), message.getTaskId(), message.getModuleType()); return false; @@ -70,14 +94,26 @@ public class TaskFileJobPublisher { if (failures >= threshold) { long nextOpenUntil = System.currentTimeMillis() + Math.max(1000L, mqCircuitOpenMillis); circuitOpenUntilMillis.set(nextOpenUntil); + countDegraded(REASON_SEND_FAILED); log.warn("[task-file-job] publish RocketMQ failed, circuit opened and fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={} openMs={} msg={}", message.getJobId(), message.getTaskId(), message.getModuleType(), failures, Math.max(1000L, mqCircuitOpenMillis), ex.getMessage()); } else { + countDegraded(REASON_SEND_FAILED); log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={}/{} msg={}", message.getJobId(), message.getTaskId(), message.getModuleType(), failures, threshold, ex.getMessage()); } return false; } } + + private void countDegraded(String reason) { + if (meterRegistry == null) { + return; + } + Counter.builder(DEGRADED_METRIC) + .tag("reason", reason) + .register(meterRegistry) + .increment(); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisherDegradedCounterTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisherDegradedCounterTest.java new file mode 100644 index 000000000..48d546893 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisherDegradedCounterTest.java @@ -0,0 +1,202 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.apache.rocketmq.spring.core.RocketMQTemplate; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * RocketMQ 降级路径的**可巡检计数出口**契约:{@code task.file.job.mq.degraded} 按 {@code reason} + * 分两档({@code template-missing} / {@code send-failed}),三处回落本地 worker 的降级点各自计数, + * 而成功发送一条都不许计。 + * + *

三条判据各挡一种蒙混:只计一档 ⇒ 反向钉(reason 用错)红;把两处 send-failed 合成一处计 + * ⇒ 阈值前后累计那条红;无条件无脑计 ⇒ 成功用例的注册表非空那条红。 + * + *

另一半契约是**现网零行为变更**:没有 MeterRegistry(1 参构造 / ObjectProvider 取不到 / 显式传 null) + * 时必须照旧返回 false、照旧开熔断,绝不因为"没指标"而改判据。 + * + *

重跑:{@code mvn -o test -Dtest=TaskFileJobPublisherDegradedCounterTest}(cwd = backend-java,纯 mock 不连库)。 + */ +class TaskFileJobPublisherDegradedCounterTest { + + private static final String METRIC = "task.file.job.mq.degraded"; + private static final String REASON_TEMPLATE_MISSING = "template-missing"; + private static final String REASON_SEND_FAILED = "send-failed"; + + @SuppressWarnings("unchecked") + private static ObjectProvider templateProvider(RocketMQTemplate template) { + ObjectProvider provider = mock(ObjectProvider.class); + when(provider.getIfAvailable()).thenReturn(template); + return provider; + } + + @SuppressWarnings("unchecked") + private static ObjectProvider registryProvider(MeterRegistry registry) { + ObjectProvider provider = mock(ObjectProvider.class); + when(provider.getIfAvailable()).thenReturn(registry); + return provider; + } + + private static TaskFileJobPublisher configured(TaskFileJobPublisher publisher, int failureThreshold) { + ReflectionTestUtils.setField(publisher, "mqEnabled", true); + ReflectionTestUtils.setField(publisher, "topic", "aiimage-result-file-job"); + ReflectionTestUtils.setField(publisher, "mqSendTimeoutMillis", 1000L); + ReflectionTestUtils.setField(publisher, "mqFailureThreshold", failureThreshold); + ReflectionTestUtils.setField(publisher, "mqCircuitOpenMillis", 60000L); + return publisher; + } + + private static TaskFileJobPublisher publisher(MeterRegistry registry, RocketMQTemplate template, + int failureThreshold) { + return configured(new TaskFileJobPublisher(templateProvider(template), registry), failureThreshold); + } + + private static RocketMQTemplate failingTemplate() { + RocketMQTemplate template = mock(RocketMQTemplate.class); + when(template.syncSend(anyString(), any(Object.class), anyLong())) + .thenThrow(new RuntimeException("broker down")); + return template; + } + + private static TaskFileJobMessage message(long jobId) { + TaskFileJobMessage message = new TaskFileJobMessage(); + message.setJobId(jobId); + message.setTaskId(900L + jobId); + message.setModuleType("similarasin"); + return message; + } + + private static double count(MeterRegistry registry, String reason) { + Counter counter = registry.find(METRIC).tag("reason", reason).counter(); + return counter == null ? 0.0d : counter.count(); + } + + private static long circuitOpenUntilMillis(TaskFileJobPublisher publisher) { + AtomicLong openUntil = (AtomicLong) ReflectionTestUtils.getField(publisher, "circuitOpenUntilMillis"); + return openUntil == null ? 0L : openUntil.get(); + } + + // -------------------------------------------------- 降级点 1:RocketMQTemplate 取不到 + + @Test + void missingTemplateCountsDegradedOnceUnderTemplateMissingReason() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + TaskFileJobPublisher publisher = publisher(registry, null, 3); + + assertThat(publisher.publish(message(1L))).as("取不到 template 仍回落本地 worker").isFalse(); + + assertThat(count(registry, REASON_TEMPLATE_MISSING)).isEqualTo(1.0d); + assertThat(count(registry, REASON_SEND_FAILED)).as("这条降级不是发送失败,reason 不许串档").isZero(); + } + + // -------------------------------------------------- 降级点 2:发送失败但未达阈值 + + @Test + void sendFailureBelowThresholdCountsDegradedUnderSendFailedReason() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 3); + + assertThat(publisher.publish(message(2L))).as("发送失败仍回落本地 worker").isFalse(); + + assertThat(count(registry, REASON_SEND_FAILED)).isEqualTo(1.0d); + assertThat(count(registry, REASON_TEMPLATE_MISSING)).isZero(); + assertThat(circuitOpenUntilMillis(publisher)).as("未达阈值不许提前开熔断").isZero(); + } + + // -------------------------------------------------- 降级点 3:发送失败并把熔断打开 + + @Test + void sendFailureThatOpensTheCircuitAlsoCountsDegradedUnderSendFailedReason() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 1); + + assertThat(publisher.publish(message(3L))).isFalse(); + + assertThat(count(registry, REASON_SEND_FAILED)).isEqualTo(1.0d); + assertThat(circuitOpenUntilMillis(publisher)).as("熔断打开那条也得被数得到") + .isGreaterThan(System.currentTimeMillis()); + } + + /** + * 两处 send-failed 分支**各自** increment:同一个 publisher 先走"未达阈值"再走"熔断打开", + * 计数必须累到 2。把两条合成一条计(或只在 catch 入口计一次)在这里红。 + */ + @Test + void bothSendFailureBranchesEachIncrementTheSameReason() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + TaskFileJobPublisher publisher = publisher(registry, failingTemplate(), 2); + + assertThat(publisher.publish(message(4L))).isFalse(); + assertThat(count(registry, REASON_SEND_FAILED)).as("未达阈值那条应计 1").isEqualTo(1.0d); + assertThat(circuitOpenUntilMillis(publisher)).isZero(); + + assertThat(publisher.publish(message(4L))).isFalse(); + assertThat(count(registry, REASON_SEND_FAILED)).as("熔断打开那条应再计 1,累计 2").isEqualTo(2.0d); + assertThat(circuitOpenUntilMillis(publisher)).isGreaterThan(System.currentTimeMillis()); + assertThat(registry.find(METRIC).tag("reason", REASON_TEMPLATE_MISSING).counter()) + .as("两处都归 send-failed,不许冒出第三个 reason").isNull(); + } + + // -------------------------------------------------- 反向钉:成功路径一次都不许多计 + + @Test + void successfulPublishIncrementsNothing() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + TaskFileJobPublisher publisher = publisher(registry, mock(RocketMQTemplate.class), 3); + + assertThat(publisher.publish(message(5L))).as("发送成功应回 true(语义未变)").isTrue(); + + assertThat(registry.getMeters()).as("成功路径不许注册任何降级指标").isEmpty(); + assertThat(registry.find(METRIC).counter()).isNull(); + assertThat(circuitOpenUntilMillis(publisher)).isZero(); + } + + // -------------------------------------------------- 零行为变更:没有注册表也要照常回落 + + @Test + void nullMeterRegistryKeepsTheDegradedSemanticsAndStillFallsBack() { + TaskFileJobPublisher withoutRegistry = publisher(null, null, 3); + assertThat(withoutRegistry.publish(message(6L))).as("无注册表时降级判定不变").isFalse(); + + TaskFileJobPublisher legacyOneArgCtor = + configured(new TaskFileJobPublisher(templateProvider(failingTemplate())), 1); + assertThat(legacyOneArgCtor.publish(message(7L))).as("1 参构造(旧签名)行为不变").isFalse(); + assertThat(circuitOpenUntilMillis(legacyOneArgCtor)).as("无注册表也要照常开熔断") + .isGreaterThan(System.currentTimeMillis()); + } + + @Test + void degradedCounterIsTakenFromTheObjectProviderChannel() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + TaskFileJobPublisher publisher = configured( + new TaskFileJobPublisher(templateProvider(null), registryProvider(registry)), 3); + + assertThat(publisher.publish(message(8L))).isFalse(); + + assertThat(count(registry, REASON_TEMPLATE_MISSING)).as("生产构造要走 ObjectProvider 取注册表") + .isEqualTo(1.0d); + } + + @Test + void emptyObjectProviderFallsBackToNoRegistryWithoutChangingBehavior() { + TaskFileJobPublisher publisher = configured( + new TaskFileJobPublisher(templateProvider(failingTemplate()), registryProvider(null)), 3); + + assertThat(publisher.publish(message(9L))).isFalse(); + assertThat(circuitOpenUntilMillis(publisher)).as("取不到注册表时熔断照旧").isZero(); + } +}