fix(brand-check): 连接未建立即收口整批重试阶梯,并留 SERVICE_UNREACHABLE 锚点
现网读数(主机B java-server,窗口 10-21 07:05→10-22 18:26):B→A:16890 自 10-21 16:30:14 起一次也没通过(A 侧 journal 里来自 121.196.149.225 的调用 10-20 1283 次、 10-21 2033 次、10-22 零次;B17 摘掉兜底 DROP 只恢复了 80/443,16890 从没在放行清单里), 而这一侧的日志是 请求异常将重试 err=I/O error … 16890 **14,944 条**、 重试耗尽判定查询失败 同因 **7,667 条**——每个品牌都要烧满 10 轮(约 45~65 秒)才知道 同一个"连不上",而外观专利正在 B 上跑批(taskId 31891),交付表因此成片「商标查询失败」。 改的是白等,不是结论:批内连续两次连接未建立且中间没有任何一次成功 ⇒ 其余品牌不再重试, 每个品牌仍真发得出它那一次首轮请求(与既有 upstreamRejecting 同一条保守口径), 判失败的行一字不变,只是 reason 从空串写成「品牌检测服务不可达(连接未建立,非本行数据问题)」。 isConnectFailure 只认 ConnectException/NoRouteToHostException/UnknownHostException (顺着 cause 链,Spring 的 ResourceAccessException 把真凶挂在下面,JDK 的 HttpConnectTimeoutException 本就是 ConnectException 子类);读超时与 5xx 故意不算—— 那是"连上了但这次没答好",重试救得回来。 用例 2 支:确定性判据用真闭环(指向一个确定没人监听的端口)——变异验证把证据阈值改成 1000000(等于不短路)后该用例判红、实测 38.64s,收口后 <5s;另一支钉 predicate 的 正例/反例(含 null 与别处的 500)。后端面整量 4691 例/0 红/0 错/17 skip/BUILD SUCCESS。
This commit is contained in:
+76
-2
@@ -14,6 +14,9 @@ import org.springframework.web.client.RestClient;
|
||||
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -53,6 +56,20 @@ public class BrandCheckClient {
|
||||
private static final String REASON_RESPONSE_TIMEOUT = "品牌检查响应超时";
|
||||
private static final String REASON_INTERRUPTED = "品牌检查被中断";
|
||||
private static final String REASON_UNEXPECTED_FAILURE = "品牌检查异常";
|
||||
/**
|
||||
* 「连接根本没建立」的证据条数:到这一数就收口整批的重试阶梯。
|
||||
*
|
||||
* <p>取 2 而不是 1:一次 refused/reset 可能是抖动,两次同形状、中间没有任何一次成功,
|
||||
* 才是服务在掉。与既有 {@code upstreamRejecting} 同一条保守口径——短路只省重试,
|
||||
* 每个品牌仍然真发得出它那一次首轮请求。
|
||||
*
|
||||
* <p>2026-10-22 现网:主机B→主机A:16890 被安全组挡死(B17 那次只恢复了 80/443),
|
||||
* 35 小时窗口里 Java 侧为这一形状发出去 {@code 请求异常将重试} <b>14,944</b> 次、
|
||||
* {@code 重试耗尽判定查询失败} <b>7,667</b> 次,而外观专利正在 B 上跑批
|
||||
* (taskId 31891)——用户看到的是交付表里成片的「商标查询失败」。
|
||||
*/
|
||||
private static final int CONNECT_FAILURE_EVIDENCE = 2;
|
||||
private static final String REASON_SERVICE_UNREACHABLE = "品牌检测服务不可达(连接未建立,非本行数据问题)";
|
||||
|
||||
/** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */
|
||||
public static final String NO_PROXY_REASON = "用户未配置代理";
|
||||
@@ -258,6 +275,9 @@ public class BrandCheckClient {
|
||||
// 一次批量一份判定:本批里只要有一个品牌跑满阶梯仍被判查询失败,其余品牌就不再
|
||||
// 各自烧完整条重试阶梯(见 checkOneBrand 的短路分支)。
|
||||
AtomicBoolean upstreamRejecting = new AtomicBoolean(false);
|
||||
// 连接层证据跨品牌累计:批内任一次成功响应清零(见 checkOneBrand)。
|
||||
// 一次批量一份,不跨批复用——另一批的 upstreamRejecting 也是这个理由。
|
||||
AtomicInteger connectFailures = new AtomicInteger();
|
||||
for (String brand : brands) {
|
||||
AtomicBoolean started = new AtomicBoolean(false);
|
||||
AtomicBoolean rejected = new AtomicBoolean(false);
|
||||
@@ -267,7 +287,8 @@ public class BrandCheckClient {
|
||||
future = CompletableFuture.supplyAsync(
|
||||
() -> {
|
||||
started.set(true);
|
||||
return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId);
|
||||
return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting,
|
||||
proxyUrl, userId, connectFailures);
|
||||
},
|
||||
checkExecutor);
|
||||
} finally {
|
||||
@@ -296,11 +317,13 @@ public class BrandCheckClient {
|
||||
}
|
||||
|
||||
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos,
|
||||
AtomicBoolean upstreamRejecting, String proxyUrl, Long userId) {
|
||||
AtomicBoolean upstreamRejecting, String proxyUrl, Long userId,
|
||||
AtomicInteger connectFailures) {
|
||||
int attempts = Math.max(1, properties.getRetryTimes());
|
||||
BrandCheckResponse response = null;
|
||||
Exception lastFailure = null;
|
||||
String reason = "";
|
||||
boolean connectDown = false;
|
||||
// 只有「把整条重试阶梯走完」才算拿到"上游在确定性拒绝"的证据。
|
||||
// 被预算或短路掐断的轮次不是证据,拿它去短路同批其余品牌会把偶发抖动放大成整批放弃。
|
||||
boolean ladderCompleted = true;
|
||||
@@ -325,11 +348,33 @@ public class BrandCheckClient {
|
||||
brand, attempt, attempts);
|
||||
break;
|
||||
}
|
||||
// 与上一条同构,但证据在连接层:会话根本没建立时重试阶梯救不回来(见常量注释的现网读数)
|
||||
if (attempt > 1 && connectFailures.get() >= CONNECT_FAILURE_EVIDENCE) {
|
||||
ladderCompleted = false;
|
||||
connectDown = true;
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 同批已连续 {} 次连接未建立,本品牌停止重试"
|
||||
+ " brand={} attempt={}/{}",
|
||||
CONNECT_FAILURE_EVIDENCE, brand, attempt, attempts);
|
||||
break;
|
||||
}
|
||||
try {
|
||||
response = postCheck(brand, strategy, userId, proxyUrl);
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
response = null;
|
||||
if (isConnectFailure(ex)) {
|
||||
if (connectFailures.incrementAndGet() >= CONNECT_FAILURE_EVIDENCE) {
|
||||
ladderCompleted = false;
|
||||
connectDown = true;
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 连续 {} 次连接未建立,判定品牌检测服务不可达,"
|
||||
+ "停止重试 brand={} attempt={}/{} err={}",
|
||||
CONNECT_FAILURE_EVIDENCE, brand, attempt, attempts, ex.getMessage());
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败):连接层证据不作数
|
||||
connectFailures.set(0);
|
||||
}
|
||||
if (attempt < attempts) {
|
||||
long retryDelayMillis = retryDelayMillis(attempt);
|
||||
log.warn("[brand-check] 请求异常将重试 brand={} attempt={}/{} 等待={}ms err={}",
|
||||
@@ -338,6 +383,8 @@ public class BrandCheckClient {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// 拿到响应=连接是通的,证据清零,别跨到下一次真断连时凑数
|
||||
connectFailures.set(0);
|
||||
if (response != null && nullToEmpty(response.getQueryFaildData()).isEmpty()) {
|
||||
// 本次查询完成且无失败项:命中侵权(faild_data)属有效结论,不重试。
|
||||
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of(), "");
|
||||
@@ -369,6 +416,13 @@ public class BrandCheckClient {
|
||||
&& !nullToEmpty(response.getQueryFaildData()).isEmpty()) {
|
||||
upstreamRejecting.set(true);
|
||||
}
|
||||
if (connectDown) {
|
||||
// 结论与旧版一致(这个品牌的商标维没核上),差别只在耗时与归因文案:
|
||||
// 品牌侧不再把一个"服务没连上"写成空 reason,交付表与补测队列都读得到真因
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 品牌检测服务不可达,按查询失败收口 brand={} lastErr={}",
|
||||
brand, lastFailure == null ? "" : lastFailure.getMessage());
|
||||
return queryFailedOutcome(brand, REASON_SERVICE_UNREACHABLE);
|
||||
}
|
||||
log.warn("[brand-check] 重试耗尽判定查询失败 brand={} attempts={} lastErr={} reason={}",
|
||||
brand, attempts,
|
||||
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage(), reason);
|
||||
@@ -490,6 +544,26 @@ public class BrandCheckClient {
|
||||
return new BrandCheckOutcome(List.of(), List.of(brand), reason);
|
||||
}
|
||||
|
||||
/**
|
||||
* 「会话根本没建立」的形状:连接被拒、无路由、域名解不开。
|
||||
*
|
||||
* <p>顺着 cause 链找:Spring 的 {@code ResourceAccessException} 把真凶挂在 cause 上
|
||||
* (现网日志读到的就是它那句 {@code I/O error on POST request for "..."}),
|
||||
* 而 JDK 的 {@code HttpConnectTimeoutException} 本身就是 {@code ConnectException} 的子类。
|
||||
*
|
||||
* <p>故意不含读超时与 5xx:那两种是"连上了但这次没答好",重试救得回来;把它们算进
|
||||
* 不可达证据,就会把一次上游抖动误判成服务在掉、把同批其余品牌的重试一起掐掉。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单个品牌的结论。
|
||||
*
|
||||
|
||||
+78
@@ -8,7 +8,13 @@ import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.ConnectException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.ServerSocket;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.net.http.HttpTimeoutException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
@@ -16,6 +22,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.springframework.web.client.ResourceAccessException;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
@@ -167,6 +175,76 @@ class BrandCheckClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接根本没建立时,重试阶梯救不回来 —— 2026-10-22 现网那一刀的形状。
|
||||
*
|
||||
* <p>主机B→主机A:16890 被安全组挡死(B17 那次摘 DROP 只恢复了 80/443),35 小时窗口里
|
||||
* {@code 请求异常将重试} 14,944 次、{@code 重试耗尽判定查询失败} 7,667 次,而外观专利
|
||||
* 正在 B 上跑批:每个品牌烧满 10 轮(约 45~65 秒)只为了得到同一个「连不上」。
|
||||
* 收口的判据是证据条数(批内连续两次连接失败、中间没有任何一次成功),
|
||||
* 结论一字不变——每条品牌仍全部判查询失败,只是不再白等,且 reason 写明是服务不可达。
|
||||
*/
|
||||
@Test
|
||||
void connectRefusalsStopTheWholeBatchFromBurningRetryLadders() throws Exception {
|
||||
BrandCheckClient client = newClientToClosedPort(10, 300, 60_000);
|
||||
List<String> brands = List.of("b1", "b2", "b3", "b4", "b5", "b6");
|
||||
|
||||
long startedAt = System.nanoTime();
|
||||
BrandCheckClient.BrandCheckBatchResult result = client.checkAll(brands, "Terms", 60_000L, 7L);
|
||||
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
|
||||
|
||||
assertThat(result.queryFaildData()).as("短路只省白等,不改结论:6 个品牌仍全部判查询失败")
|
||||
.hasSize(brands.size());
|
||||
assertThat(result.reason()).as("归因必须写明「服务没连上」,不许再留空 reason").contains("品牌检测服务不可达");
|
||||
// 不短路时:6 品牌按并发 3 分两波,每波阶梯 300×(1+…+9)≈13.5 秒 ≈ 27 秒;
|
||||
// 判据取 5 秒——远低于"烧满"的最坏读数,又给机器负载留了十倍余量。
|
||||
assertThat(elapsedMillis).as("连接层证据到两次即收口,实测耗时=%dms", elapsedMillis)
|
||||
.isLessThan(5_000L);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判据必须只认"会话没建立"这一类。读超时与 5xx 是"连上了但这次没答好",
|
||||
* 把它们算进不可达证据,一次上游抖动就会掐掉同批别的品牌本该拿到的重试。
|
||||
*/
|
||||
@Test
|
||||
void connectFailurePredicateSeparatesNoConnectionFromBadAnswer() {
|
||||
assertThat(BrandCheckClient.isConnectFailure(
|
||||
new ConnectException("Connection refused: connect"))).isTrue();
|
||||
assertThat(BrandCheckClient.isConnectFailure(
|
||||
new UnknownHostException("brand.aishufu.top"))).isTrue();
|
||||
assertThat(BrandCheckClient.isConnectFailure(new NoRouteToHostException("no route"))).isTrue();
|
||||
// Spring 把真凶挂在 cause 上,现网日志读到的就是外层那句 I/O error
|
||||
assertThat(BrandCheckClient.isConnectFailure(new ResourceAccessException(
|
||||
"I/O error on POST request for \"http://127.0.0.1:16890/brand_check\"",
|
||||
new ConnectException("Connection refused")))).isTrue();
|
||||
|
||||
assertThat(BrandCheckClient.isConnectFailure(
|
||||
new SocketTimeoutException("Read timed out"))).isFalse();
|
||||
assertThat(BrandCheckClient.isConnectFailure(
|
||||
new HttpTimeoutException("request timed out"))).isFalse();
|
||||
assertThat(BrandCheckClient.isConnectFailure(
|
||||
new IllegalStateException("500 Internal Server Error"))).isFalse();
|
||||
assertThat(BrandCheckClient.isConnectFailure(null)).isFalse();
|
||||
}
|
||||
|
||||
/** 一个确定没人监听的端口:connect 当场被拒,正是"服务不在了"而不是"服务很慢"。 */
|
||||
private static int closedPort() throws IOException {
|
||||
try (ServerSocket socket = new ServerSocket(0)) {
|
||||
return socket.getLocalPort();
|
||||
}
|
||||
}
|
||||
|
||||
private BrandCheckClient newClientToClosedPort(int retryTimes, int retryIntervalMillis,
|
||||
int totalTimeoutMillis) throws IOException {
|
||||
BrandCheckProperties props = new BrandCheckProperties();
|
||||
props.setBaseUrl("http://127.0.0.1:" + closedPort());
|
||||
props.setPath("/brand_check");
|
||||
props.setRetryTimes(retryTimes);
|
||||
props.setRetryIntervalMillis(retryIntervalMillis);
|
||||
props.setTotalTimeoutMillis(totalTimeoutMillis);
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets());
|
||||
}
|
||||
|
||||
private BrandCheckClient newClient(HttpServer server, int retryTimes) {
|
||||
BrandCheckProperties props = new BrandCheckProperties();
|
||||
props.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
|
||||
Reference in New Issue
Block a user