fix(实例路由): 转发失败不再伪装成「任务不存活」
归属节点滚动重启时跨节点转发会连接失败,原来返回 ApiResponse.fail(40903) —— HTTP 200 + data:null。客户端把「拿不到数据」解析成 alive=false,于是把健康的长 任务主动停掉(2026-09-18 任务 28616:跑到 66/253 被自杀,只跑到第 2/6 页)。 改为 503 + 空 body:新客户端按状态码判未知继续跑;老客户端因 body 不是 JSON、 resp.json() 抛异常也落到未知——两边都不会再把未知当成死。顺带让这类故障在 HTTP 指标里可见(原先记成 200,滚动重启期间丢了多少心跳监控完全看不到)。
This commit is contained in:
+10
-1
@@ -55,10 +55,19 @@ public class GlobalExceptionHandler {
|
||||
? ApiResponse.fail(forwardEx.getMessage())
|
||||
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
||||
} catch (Exception forwardEx) {
|
||||
// 转发失败是**瞬时基础设施故障**(归属实例正在滚动重启),不是业务结论,
|
||||
// 更不能表达成「任务不存活」。原先返回 ApiResponse.fail(40903) —— HTTP 200
|
||||
// 加 data:null,而客户端那句 bool((resp.json().get("data") or {}).get("alive"))
|
||||
// 会把「拿不到数据」折叠成 alive=false,于是客户端把**健康的长任务主动停掉**:
|
||||
// 2026-09-18 任务 28616 就是这么死的(归属节点 server-110 重启窗口内,心跳经
|
||||
// nginx 落到 server-121,转发 3 次 Connection refused 后返回空 data)。
|
||||
// 改为 503 + 空 body:新客户端按状态码判为「未知」继续跑;老客户端因 body 不是
|
||||
// JSON、resp.json() 抛异常,同样落到「未知」。顺带让这类故障在 HTTP 指标里可见
|
||||
// (原先记成 200,监控完全看不到滚动重启期间丢了多少心跳)。
|
||||
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
||||
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
||||
forwardEx.getMessage(), forwardEx);
|
||||
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage());
|
||||
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.nanri.aiimage.common.exception;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.ResourceAccessException;
|
||||
|
||||
import java.net.ConnectException;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 跨实例转发的失败语义。
|
||||
*
|
||||
* <p>2026-09-18 任务 28616:归属节点 server-110 滚动重启期间,客户端心跳经 nginx 落到
|
||||
* server-121,转发 3 次 Connection refused。当时这里返回 {@code ApiResponse.fail(40903)}
|
||||
* ——HTTP 200 + {@code data:null},而客户端用
|
||||
* {@code bool((resp.json().get("data") or {}).get("alive"))} 解析,把「拿不到数据」折叠成
|
||||
* {@code alive=false},于是客户端把一个跑到 66/253 的健康上架任务主动停掉、关闭店铺。
|
||||
* 修复后转发失败返回 503 + 空 body:新客户端按状态码判为「未知」,老客户端因 body 不是
|
||||
* JSON、解析抛异常同样落到「未知」,两边都不会再自杀。</p>
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class GlobalExceptionHandlerTest {
|
||||
|
||||
@Mock private TaskOwnerForwardService taskOwnerForwardService;
|
||||
|
||||
@InjectMocks private GlobalExceptionHandler handler;
|
||||
|
||||
private static TaskOwnerMismatchException ownerMismatch() {
|
||||
return new TaskOwnerMismatchException(28616L, "PUBLISH task heartbeat", "server-110", "server-121");
|
||||
}
|
||||
|
||||
@Test
|
||||
void forwardConnectFailureReturns503WithEmptyBody() {
|
||||
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||
.thenThrow(new ResourceAccessException("Connection refused",
|
||||
new ConnectException("Connection refused")));
|
||||
|
||||
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||
|
||||
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
|
||||
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
|
||||
// 空 body 是关键:一旦带上 JSON,老客户端的 bool((data or {}).get("alive")) 又会判成「死」
|
||||
assertNull(response.getBody(), "转发失败必须无响应体,否则老客户端会把未知当成任务已死");
|
||||
}
|
||||
|
||||
@Test
|
||||
void successfulForwardPassesThroughUpstreamStatusAndBody() {
|
||||
byte[] upstream = "{\"success\":true,\"data\":{\"alive\":true}}".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||
.thenReturn(ResponseEntity.ok(upstream));
|
||||
|
||||
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||
|
||||
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
assertEquals(upstream, response.getBody(), "转发成功时上游响应体必须原样透传");
|
||||
}
|
||||
|
||||
@Test
|
||||
void configErrorKeepsBusinessEnvelopeInsteadOfServiceUnavailable() {
|
||||
// 路由未配置 / 检测到转发循环属于配置错误,不是瞬时故障:保留业务信封,不回 503
|
||||
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||
.thenThrow(new BusinessException(40903, "任务归属实例未配置服务路由:server-110"));
|
||||
|
||||
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||
|
||||
ApiResponse<?> response = assertInstanceOf(ApiResponse.class, result);
|
||||
assertFalse(response.isSuccess(), "配置错误仍按业务失败返回");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user