后端稳定性建设｜容错四大金刚：重试（上）线上 Feign 重试故障深度复盘

feign.RetryableException: Failed to connect to /172.19.115.10:8080 executing POST http/data-rpc-service/data/batchQuery
	at feign.FeignException.errorExecuting(FeignException.java:268)
	at feign.SynchronousMethodHandler.executeAndDecode(SynchronousMethodHandler.java:129)
	at feign.SynchronousMethodHandler.invoke(SynchronousMethodHandler.java:89)
	at feign.ReflectiveFeign$FeignInvocationHandler.invoke(ReflectiveFeign.java:100)
	at org.springframework.cloud.openfeign.FeignCachingInvocationHandlerFactory$1.proceed(FeignCachingInvocationHandlerFactory.java:66)
	at org.springframework.cache.interceptor.CacheInterceptor.lambda$invoke$0(CacheInterceptor.java:54)
	at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:351)
	at org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:64)
	at org.springframework.cloud.openfeign.FeignCachingInvocationHandlerFactory.lambda$create$1(FeignCachingInvocationHandlerFactory.java:53)
	at jdk.proxy3.$Proxy240.batchQuery(Unknown Source)
        // 上层业务链路
	at com.xxx.search.core.infra.service.DataGatewayImpl.batchQuery
	at com.xxx.search.core.application.biz.impl.DataServiceImpl$RebuildTask.run
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
	at java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at com.alibaba.ttl.TtlRunnable.run(TtlRunnable.java:60)
	at org.dromara.dynamictp.core.support.task.runnable.MdcRunnable.run
	at org.dromara.dynamictp.core.support.task.runnable.DtpRunnable.run
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.lang.Thread.run(Thread.java:959)
Caused by: java.net.ConnectException: Failed to connect to /172.19.115.10:8080
	at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:297)
	at okhttp3.internal.connection.RealConnection.connect(RealConnection.kt:207)
	at okhttp3.internal.connection.ExchangeFinder.findConnection(ExchangeFinder.kt:226)
	at okhttp3.internal.connection.ExchangeFinder.findHealthConnection(ExchangeFinder.kt:106)
	at okhttp3.internal.connection.ExchangeFinder.find(ExchangeFinder.kt:74)
	at okhttp3.internal.connection.RealCall.initExchange$okhttp(RealCall.kt:255)
	at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.kt:32)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:109)
	at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.kt:95)
	at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.kt:83)
	at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.kt:76)
	at com.framework.rpc.feign.interceptor.MetricInterceptor.intercept
	at com.framework.rpc.feign.interceptor.ProtocolInterceptor.intercept
	at io.opentelemetry.javaagent.shaded.instrumentation.okhttp.v3_0.internal.ConnectionErrorSpanInterceptor.intercept
	at okhttp3.internal.connection.RealCall.getResponseWithInterceptorChain$okhttp(RealCall.kt:201)
	at okhttp3.internal.connection.RealCall.execute(RealCall.kt:154)
	at feign.okhttp.OkHttpClient.execute(OkHttpClient.java:172)
	at org.springframework.cloud.openfeign.loadbalancer.RetryableFeignBlockingLoadBalancerClient.lambda$execute$2
	at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:329)
	at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:225)
	at org.springframework.cloud.openfeign.loadbalancer.RetryableFeignBlockingLoadBalancerClient.execute
	at feign.SynchronousMethodHandler.executeAndDecode(SynchronousMethodHandler.java:119)
	... 22 more
Suppressed: java.io.IOException: unexpected end of stream on http://172.19.115.10:8080/...
	at okhttp3.internal.http1.Http1ExchangeCodec.readResponseHeaders(Http1ExchangeCodec.kt:202)
	at okhttp3.internal.connection.Exchange.readResponseHeaders(Exchange.kt:106)
	at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.kt:79)
Caused by: java.io.EOFException: \n not found: limit=0 content=…
	at okio.RealBufferedSource.readUtf8LineStrict(RealBufferedSource.kt:332)
	at okhttp3.internal.http1.HeadersReader.readLine(HeadersReader.kt:29)
	at okhttp3.internal.http1.Http1ExchangeCodec.readResponseHeaders(Http1ExchangeCodec.kt:178)
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method)
	at sun.nio.ch.NioSocketImpl.timedFinishConnect(NioSocketImpl.java:549)
	at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:295)
// 底层重复连接堆栈省略

while (true) {
    try {
        return executeAndDecode(template, options);
    } catch (RetryableException e) {
        retryer.continueOrPropagate(e);
        // 复用原有请求模板，目标IP固定不变
        continue;
    }
}

public boolean canRetry(LoadBalancedRetryContext context) {
    HttpMethod method = context.getRequest().getMethod();
    // 默认retryOnAllOperations=false，仅GET允许重试
    return HttpMethod.GET.equals(method) || properties.getRetry().isRetryOnAllOperations();
}