Detect connection timeouts in Hedging

src/Libraries/Microsoft.Extensions.Http.Resilience/Hedging/HttpClientHedgingResiliencePredicates.cs

@@ -2,10 +2,13 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Diagnostics.CodeAnalysis;
 using System.Net.Http;
+using Microsoft.Shared.DiagnosticIds;
 using Microsoft.Shared.Diagnostics;
 using Polly;
 using Polly.CircuitBreaker;
+using Polly.Hedging;
 
 namespace Microsoft.Extensions.Http.Resilience;
 
@@ -18,12 +21,28 @@ public static class HttpClientHedgingResiliencePredicates
     /// Determines whether an outcome should be treated by hedging as a transient failure.
     /// </summary>
     /// <returns><see langword="true"/> if outcome is transient, <see langword="false"/> if not.</returns>
-    public static bool IsTransient(Outcome<HttpResponseMessage> outcome) => outcome switch
-    {
-        { Result: { } response } when HttpClientResiliencePredicates.IsTransientHttpFailure(response) => true,
-        { Exception: { } exception } when IsTransientHttpException(exception) => true,
-        _ => false,
-    };
+    public static bool IsTransient(Outcome<HttpResponseMessage> outcome)
+        => outcome switch
+        {
+            { Result: { } response } when HttpClientResiliencePredicates.IsTransientHttpFailure(response) => true,
+            { Exception: { } exception } when IsTransientHttpException(exception) => true,
+            _ => false,
+        };
+
+    /// <summary>
+    /// Determines whether an <see cref="HttpResponseMessage"/> should be treated by hedging as a transient failure.
+    /// </summary>
+    /// <param name="args">A <see cref="HedgingPredicateArguments{T}"/>.</param>
+    /// <returns><see langword="true"/> if outcome is transient, <see langword="false"/> if not.</returns>
+    [Experimental(diagnosticId: DiagnosticIds.Experiments.Resilience, UrlFormat = DiagnosticIds.UrlFormat)]
+    public static bool IsTransient(HedgingPredicateArguments<HttpResponseMessage> args)
+        => IsConnectionTimeout(args)
+        || IsTransient(args.Outcome);
+
+    internal static bool IsConnectionTimeout(HedgingPredicateArguments<HttpResponseMessage> args)
+        => !args.Context.CancellationToken.IsCancellationRequested
+        && args.Outcome.Exception is OperationCanceledException { Source: "System.Private.CoreLib" }
+        && args.Outcome.Exception.InnerException is TimeoutException;
 
     /// <summary>
     /// Determines whether an exception should be treated by hedging as a transient failure.
@@ -35,8 +54,7 @@ internal static bool IsTransientHttpException(Exception exception)
         return exception switch
         {
             BrokenCircuitException => true,
-            _ when HttpClientResiliencePredicates.IsTransientHttpException(exception) => true,
-            _ => false,
+            _ => HttpClientResiliencePredicates.IsTransientHttpException(exception),
         };
     }
 }

src/Libraries/Microsoft.Extensions.Http.Resilience/Hedging/HttpHedgingStrategyOptions.cs

@@ -21,6 +21,6 @@ public class HttpHedgingStrategyOptions : HedgingStrategyOptions<HttpResponseMes
     /// </remarks>
     public HttpHedgingStrategyOptions()
     {
-        ShouldHandle = args => new ValueTask<bool>(HttpClientHedgingResiliencePredicates.IsTransient(args.Outcome));
+        ShouldHandle = args => new ValueTask<bool>(HttpClientHedgingResiliencePredicates.IsTransient(args));
     }
 }

test/Libraries/Microsoft.Extensions.Http.Resilience.Tests/Hedging/HedgingTests.cs

@@ -35,6 +35,7 @@ public abstract class HedgingTests<TBuilder> : IDisposable
     private readonly Func<RequestRoutingStrategy> _requestRoutingStrategyFactory;
     private readonly IServiceCollection _services;
     private readonly Queue<HttpResponseMessage> _responses = new();
+    private ServiceProvider? _serviceProvider;
     private bool _failure;
 
     private protected HedgingTests(Func<IHttpClientBuilder, Func<RequestRoutingStrategy>, TBuilder> createDefaultBuilder)
@@ -63,6 +64,11 @@ public void Dispose()
         _requestRoutingStrategyMock.VerifyAll();
         _cancellationTokenSource.Cancel();
         _cancellationTokenSource.Dispose();
+        _serviceProvider?.Dispose();
+        foreach (var response in _responses)
+        {
+            response.Dispose();
+        }
     }
 
     [Fact]
@@ -93,7 +99,7 @@ public async Task SendAsync_EnsureContextFlows()
 
         using var client = CreateClientWithHandler();
 
-        await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var _ = await client.SendAsync(request, _cancellationTokenSource.Token);
 
         Assert.Equal(2, calls);
     }
@@ -108,7 +114,7 @@ public async Task SendAsync_NoErrors_ShouldReturnSingleResponse()
 
         AddResponse(HttpStatusCode.OK);
 
-        var response = await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var _ = await client.SendAsync(request, _cancellationTokenSource.Token);
         AssertNoResponse();
 
         Assert.Single(Requests);
@@ -164,7 +170,7 @@ public async Task SendAsync_NoRoutesLeftAndSomeResultPresent_ShouldReturn()
 
         using var client = CreateClientWithHandler();
 
-        var result = await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var result = await client.SendAsync(request, _cancellationTokenSource.Token);
         Assert.Equal(DefaultHedgingAttempts + 1, Requests.Count);
         Assert.Equal(HttpStatusCode.ServiceUnavailable, result.StatusCode);
     }
@@ -183,7 +189,7 @@ public async Task SendAsync_EnsureDistinctContextForEachAttempt()
 
         using var client = CreateClientWithHandler();
 
-        await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var _ = await client.SendAsync(request, _cancellationTokenSource.Token);
 
         RequestContexts.Distinct().OfType<ResilienceContext>().Should().HaveCount(3);
     }
@@ -204,7 +210,7 @@ public async Task SendAsync_EnsureContextReplacedInRequestMessage()
 
         using var client = CreateClientWithHandler();
 
-        await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var _ = await client.SendAsync(request, _cancellationTokenSource.Token);
 
         RequestContexts.Distinct().OfType<ResilienceContext>().Should().HaveCount(3);
 
@@ -226,7 +232,7 @@ public async Task SendAsync_NoRoutesLeft_EnsureLessThanMaxHedgedAttempts()
 
         using var client = CreateClientWithHandler();
 
-        var result = await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var _ = await client.SendAsync(request, _cancellationTokenSource.Token);
         Assert.Equal(2, Requests.Count);
     }
 
@@ -244,7 +250,7 @@ public async Task SendAsync_FailedExecution_ShouldReturnResponseFromHedging()
 
         using var client = CreateClientWithHandler();
 
-        var result = await client.SendAsync(request, _cancellationTokenSource.Token);
+        using var result = await client.SendAsync(request, _cancellationTokenSource.Token);
         Assert.Equal(3, Requests.Count);
         Assert.Equal(HttpStatusCode.OK, result.StatusCode);
         Assert.Equal("https://enpoint-1:80/some-path?query", Requests[0]);
@@ -268,7 +274,12 @@ protected void AddResponse(HttpStatusCode statusCode, int count)
 
     protected abstract void ConfigureHedgingOptions(Action<HttpHedgingStrategyOptions> configure);
 
-    protected HttpClient CreateClientWithHandler() => _services.BuildServiceProvider().GetRequiredService<IHttpClientFactory>().CreateClient(ClientId);
+    protected HttpClient CreateClientWithHandler()
+    {
+        _serviceProvider?.Dispose();
+        _serviceProvider = _services.BuildServiceProvider();
+        return _serviceProvider.GetRequiredService<IHttpClientFactory>().CreateClient(ClientId);
+    }
 
     private Task<HttpResponseMessage> InnerHandlerFunction(HttpRequestMessage request, CancellationToken cancellationToken)
     {

test/Libraries/Microsoft.Extensions.Http.Resilience.Tests/Hedging/OperationCanceledExceptionMock.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.Extensions.Http.Resilience.Test.Hedging;
+
+internal sealed class OperationCanceledExceptionMock : OperationCanceledException
+{
+    public OperationCanceledExceptionMock(Exception innerException)
+        : base(null, innerException)
+    {
+    }
+
+    public override string? Source { get => "System.Private.CoreLib"; set => base.Source = value; }
+}

test/Libraries/Microsoft.Extensions.Http.Resilience.Tests/Hedging/StandardHedgingTests.cs

@@ -46,23 +46,24 @@ public void EnsureValidated_BasicValidation()
     {
         Builder.Configure(options => options.Hedging.MaxHedgedAttempts = -1);
 
-        Assert.Throws<OptionsValidationException>(() => CreateClientWithHandler());
+        Assert.Throws<OptionsValidationException>(CreateClientWithHandler);
     }
 
     [Fact]
     public void EnsureValidated_AdvancedValidation()
     {
         Builder.Configure(options => options.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(1));
 
-        Assert.Throws<OptionsValidationException>(() => CreateClientWithHandler());
+        Assert.Throws<OptionsValidationException>(CreateClientWithHandler);
     }
 
     [Fact]
     public void Configure_Callback_Ok()
     {
         Builder.Configure(o => o.Hedging.MaxHedgedAttempts = 8);
 
-        var options = Builder.Services.BuildServiceProvider().GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
+        using var serviceProvider = Builder.Services.BuildServiceProvider();
+        var options = serviceProvider.GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
 
         Assert.Equal(8, options.Hedging.MaxHedgedAttempts);
     }
@@ -76,7 +77,8 @@ public void Configure_CallbackWithServiceProvider_Ok()
             o.Hedging.MaxHedgedAttempts = 8;
         });
 
-        var options = Builder.Services.BuildServiceProvider().GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
+        using var serviceProvider = Builder.Services.BuildServiceProvider();
+        var options = serviceProvider.GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
 
         Assert.Equal(8, options.Hedging.MaxHedgedAttempts);
     }
@@ -97,15 +99,17 @@ public void Configure_ValidConfigurationSection_ShouldInitialize()
 
         Builder.Configure(section);
 
-        var options = Builder.Services.BuildServiceProvider().GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
+        using var serviceProvider = Builder.Services.BuildServiceProvider();
+        var options = serviceProvider.GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
 
         Assert.Equal(8, options.Hedging.MaxHedgedAttempts);
     }
 
     [Fact]
     public void ActionGenerator_Ok()
     {
-        var options = Builder.Services.BuildServiceProvider().GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
+        using var serviceProvider = Builder.Services.BuildServiceProvider();
+        var options = serviceProvider.GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>().Get(Builder.Name);
         var generator = options.Hedging.ActionGenerator;
         var primary = ResilienceContextPool.Shared.Get();
         var secondary = ResilienceContextPool.Shared.Get();
@@ -133,9 +137,12 @@ public void Configure_InvalidConfigurationSection_ShouldThrow()
         Builder.Configure(section);
 
         Assert.Throws<InvalidOperationException>(() =>
-            Builder.Services.BuildServiceProvider()
-            .GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>()
-            .Get(Builder.Name));
+        {
+            using var serviceProvider = Builder.Services.BuildServiceProvider();
+            return serviceProvider
+                    .GetRequiredService<IOptionsMonitor<HttpStandardHedgingResilienceOptions>>()
+                    .Get(Builder.Name);
+        });
     }
 #endif
 
@@ -163,7 +170,7 @@ public void Configure_EmptyConfigurationSection_ShouldThrow()
     [Fact]
     public void VerifyPipeline()
     {
-        var serviceProvider = Builder.Services.BuildServiceProvider();
+        using var serviceProvider = Builder.Services.BuildServiceProvider();
         var pipelineProvider = serviceProvider.GetRequiredService<ResiliencePipelineProvider<HttpKey>>();
 
         // primary handler
@@ -209,7 +216,7 @@ public async Task VerifyPipelineSelection(string? customKey)
         using var request = new HttpRequestMessage(HttpMethod.Get, "https://key:80/discarded");
         AddResponse(HttpStatusCode.OK);
 
-        var response = await client.SendAsync(request, CancellationToken.None);
+        using var response = await client.SendAsync(request, CancellationToken.None);
 
         provider.VerifyAll();
     }
@@ -235,14 +242,14 @@ public async Task DynamicReloads_Ok()
         // act && assert
         AddResponse(HttpStatusCode.InternalServerError, 3);
         using var firstRequest = new HttpRequestMessage(HttpMethod.Get, "https://to-be-replaced:1234/some-path?query");
-        await client.SendAsync(firstRequest);
+        using var _ = await client.SendAsync(firstRequest);
         AssertNoResponse();
 
         reloadAction(new() { { "standard:Hedging:MaxHedgedAttempts", "6" } });
 
         AddResponse(HttpStatusCode.InternalServerError, 7);
         using var secondRequest = new HttpRequestMessage(HttpMethod.Get, "https://to-be-replaced:1234/some-path?query");
-        await client.SendAsync(secondRequest);
+        using var __ = await client.SendAsync(secondRequest);
         AssertNoResponse();
     }
 
@@ -257,11 +264,70 @@ public async Task NoRouting_Ok()
         // act && assert
         AddResponse(HttpStatusCode.InternalServerError, 3);
         using var firstRequest = new HttpRequestMessage(HttpMethod.Get, "https://some-endpoint:1234/some-path?query");
-        await client.SendAsync(firstRequest);
+        using var _ = await client.SendAsync(firstRequest);
         AssertNoResponse();
 
         Requests.Should().AllSatisfy(r => r.Should().Be("https://some-endpoint:1234/some-path?query"));
     }
 
+    [Fact]
+    public async Task SendAsync_FailedConnect_ShouldReturnResponseFromHedging()
+    {
+        const string FailingEndpoint = "www.failing-host.com";
+
+        var services = new ServiceCollection();
+        _ = services
+            .AddHttpClient(ClientId)
+            .ConfigurePrimaryHttpMessageHandler(() => new MockHttpMessageHandler(FailingEndpoint))
+            .AddStandardHedgingHandler(routing =>
+                routing.ConfigureOrderedGroups(g =>
+                {
+                    g.Groups.Add(new UriEndpointGroup
+                    {
+                        Endpoints = [new WeightedUriEndpoint { Uri = new Uri($"https://{FailingEndpoint}:3000") }]
+                    });
+
+                    g.Groups.Add(new UriEndpointGroup
+                    {
+                        Endpoints = [new WeightedUriEndpoint { Uri = new Uri("https://microsoft.com") }]
+                    });
+                }))
+            .Configure(opt =>
+            {
+                opt.Hedging.MaxHedgedAttempts = 10;
+                opt.Hedging.Delay = TimeSpan.FromSeconds(11);
+                opt.Endpoint.CircuitBreaker.FailureRatio = 0.99;
+                opt.Endpoint.CircuitBreaker.SamplingDuration = TimeSpan.FromSeconds(900);
+                opt.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(200);
+                opt.Endpoint.Timeout.Timeout = TimeSpan.FromSeconds(200);
+            });
+
+        await using var provider = services.BuildServiceProvider();
+        var clientFactory = provider.GetRequiredService<IHttpClientFactory>();
+        using var client = clientFactory.CreateClient(ClientId);
+
+        var ex = await Record.ExceptionAsync(async () =>
+        {
+            using var _ = await client.GetAsync($"https://{FailingEndpoint}:3000");
+        });
+
+        Assert.Null(ex);
+    }
+
     protected override void ConfigureHedgingOptions(Action<HttpHedgingStrategyOptions> configure) => Builder.Configure(options => configure(options.Hedging));
+
+    private class MockHttpMessageHandler(string failingEndpoint) : HttpMessageHandler
+    {
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            if (request.RequestUri?.Host == failingEndpoint)
+            {
+                await Task.Delay(100, cancellationToken);
+                throw new OperationCanceledExceptionMock(new TimeoutException());
+            }
+
+            await Task.Delay(1000, cancellationToken);
+            return new HttpResponseMessage(HttpStatusCode.OK);
+        }
+    }
 }

test/Libraries/Microsoft.Extensions.Http.Resilience.Tests/Microsoft.Extensions.Http.Resilience.Tests.csproj

@@ -4,11 +4,6 @@
     <Description>Unit tests for Microsoft.Extensions.Http.Resilience.</Description>
   </PropertyGroup>
 
-  <PropertyGroup Condition=" '$(TargetFramework)' == 'net462' ">
-    <!-- See https://github.com/dotnet/extensions/issues/4531 -->
-    <SkipTests>true</SkipTests>
-  </PropertyGroup>
-
   <ItemGroup>
     <None Update="configs\appsettings.json" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>