fix(实例路由): 转发失败不再伪装成「任务不存活」

归属节点滚动重启时跨节点转发会连接失败,原来返回 ApiResponse.fail(40903) ——
HTTP 200 + data:null。客户端把「拿不到数据」解析成 alive=false,于是把健康的长
任务主动停掉(2026-09-18 任务 28616:跑到 66/253 被自杀,只跑到第 2/6 页)。

改为 503 + 空 body:新客户端按状态码判未知继续跑;老客户端因 body 不是 JSON、
resp.json() 抛异常也落到未知——两边都不会再把未知当成死。顺带让这类故障在
HTTP 指标里可见(原先记成 200,滚动重启期间丢了多少心跳监控完全看不到)。
This commit is contained in:
2026-09-18 16:25:33 +08:00
parent e0303f9cba
commit 46be044121
2 changed files with 95 additions and 1 deletions
@@ -55,10 +55,19 @@ public class GlobalExceptionHandler {
? ApiResponse.fail(forwardEx.getMessage()) ? ApiResponse.fail(forwardEx.getMessage())
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage()); : ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
} catch (Exception forwardEx) { } catch (Exception forwardEx) {
// 转发失败是**瞬时基础设施故障**(归属实例正在滚动重启),不是业务结论,
// 更不能表达成「任务不存活」。原先返回 ApiResponse.fail(40903) —— HTTP 200
// 加 data:null,而客户端那句 bool((resp.json().get("data") or {}).get("alive"))
// 会把「拿不到数据」折叠成 alive=false,于是客户端把**健康的长任务主动停掉**:
// 2026-09-18 任务 28616 就是这么死的(归属节点 server-110 重启窗口内,心跳经
// nginx 落到 server-121,转发 3 次 Connection refused 后返回空 data)。
// 改为 503 + 空 body:新客户端按状态码判为「未知」继续跑;老客户端因 body 不是
// JSON、resp.json() 抛异常,同样落到「未知」。顺带让这类故障在 HTTP 指标里可见
// (原先记成 200,监控完全看不到滚动重启期间丢了多少心跳)。
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}", log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
forwardEx.getMessage(), forwardEx); forwardEx.getMessage(), forwardEx);
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage()); return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build();
} }
} }
@@ -0,0 +1,85 @@
package com.nanri.aiimage.common.exception;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.ResourceAccessException;
import java.net.ConnectException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* 跨实例转发的失败语义。
*
* <p>2026-09-18 任务 28616:归属节点 server-110 滚动重启期间,客户端心跳经 nginx 落到
* server-121,转发 3 次 Connection refused。当时这里返回 {@code ApiResponse.fail(40903)}
* ——HTTP 200 + {@code data:null},而客户端用
* {@code bool((resp.json().get("data") or {}).get("alive"))} 解析,把「拿不到数据」折叠成
* {@code alive=false},于是客户端把一个跑到 66/253 的健康上架任务主动停掉、关闭店铺。
* 修复后转发失败返回 503 + 空 body:新客户端按状态码判为「未知」,老客户端因 body 不是
* JSON、解析抛异常同样落到「未知」,两边都不会再自杀。</p>
*/
@ExtendWith(MockitoExtension.class)
class GlobalExceptionHandlerTest {
@Mock private TaskOwnerForwardService taskOwnerForwardService;
@InjectMocks private GlobalExceptionHandler handler;
private static TaskOwnerMismatchException ownerMismatch() {
return new TaskOwnerMismatchException(28616L, "PUBLISH task heartbeat", "server-110", "server-121");
}
@Test
void forwardConnectFailureReturns503WithEmptyBody() {
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
.thenThrow(new ResourceAccessException("Connection refused",
new ConnectException("Connection refused")));
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
// 空 body 是关键:一旦带上 JSON,老客户端的 bool((data or {}).get("alive")) 又会判成「死」
assertNull(response.getBody(), "转发失败必须无响应体,否则老客户端会把未知当成任务已死");
}
@Test
void successfulForwardPassesThroughUpstreamStatusAndBody() {
byte[] upstream = "{\"success\":true,\"data\":{\"alive\":true}}".getBytes(java.nio.charset.StandardCharsets.UTF_8);
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
.thenReturn(ResponseEntity.ok(upstream));
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertEquals(upstream, response.getBody(), "转发成功时上游响应体必须原样透传");
}
@Test
void configErrorKeepsBusinessEnvelopeInsteadOfServiceUnavailable() {
// 路由未配置 / 检测到转发循环属于配置错误,不是瞬时故障:保留业务信封,不回 503
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
.thenThrow(new BusinessException(40903, "任务归属实例未配置服务路由:server-110"));
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
ApiResponse<?> response = assertInstanceOf(ApiResponse.class, result);
assertFalse(response.isSuccess(), "配置错误仍按业务失败返回");
}
}