add advanced cb (#89)

* add advanced cb and metrics * edit metrics * rename field * add publishing * add host to metric * add publishing * add machine name instead of host * fix metrics * fix metrics * edit readme, remove metrics and add parameter for disabling CB * edit comment * edit readme --------- Co-authored-by: koverin.a <[email protected]>
atidev · Aug 7, 2024 · 9b37263 · 9b37263
1 parent 8615548
commit 9b37263
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 39 deletions.
diff --git a/ATI.Services.Common/Http/Extensions/HttpClientBuilderPolicyExtensions.cs b/ATI.Services.Common/Http/Extensions/HttpClientBuilderPolicyExtensions.cs
@@ -4,6 +4,7 @@
 using System.Net;
 using System.Net.Http;
 using ATI.Services.Common.Logging;
+using ATI.Services.Common.Metrics;
 using ATI.Services.Common.Options;
 using Microsoft.Extensions.DependencyInjection;
 using NLog;
@@ -13,6 +14,7 @@
 using Polly.Extensions.Http;
 using Polly.Registry;
 using Polly.Timeout;
+using Prometheus;
 
 namespace ATI.Services.Common.Http.Extensions;
 
@@ -88,23 +90,19 @@ public static IHttpClientBuilder AddHostSpecificCircuitBreakerPolicy(
         var registry = new PolicyRegistry();
         return clientBuilder.AddPolicyHandler(message =>
         {
-            var circuitBreakerExceptionsCount = serviceOptions.CircuitBreakerExceptionsCount;
+            var circuitBreakerSamplingDuration = serviceOptions.CircuitBreakerSamplingDuration;
             var circuitBreakerDuration = serviceOptions.CircuitBreakerDuration;
-
-            var retryPolicySettings = message.Options.GetRetryPolicy();
-            if (retryPolicySettings != null)
-            {
-                if (retryPolicySettings.CircuitBreakerExceptionsCount != null)
-                    circuitBreakerExceptionsCount = retryPolicySettings.CircuitBreakerExceptionsCount.Value;
-                if (retryPolicySettings.CircuitBreakerDuration != null)
-                    circuitBreakerDuration = retryPolicySettings.CircuitBreakerDuration.Value;
-            }
-
-            if (circuitBreakerExceptionsCount == 0)
+            var circuitBreakerFailureThreshold = serviceOptions.CircuitBreakerFailureThreshold;
+            var circuitBreakerMinimumThroughput = serviceOptions.CircuitBreakerMinimumThroughput;
+            
+            if (!serviceOptions.CircuitBreakerEnabled)
                 return Policy.NoOpAsync<HttpResponseMessage>();
 
             var policyKey = $"{message.RequestUri.Host}:{message.RequestUri.Port}";
-            var policy = registry.GetOrAdd(policyKey, BuildCircuitBreakerPolicy(message, serviceOptions, circuitBreakerExceptionsCount, circuitBreakerDuration, logger));
+            var policy = registry.GetOrAdd(policyKey,
+                BuildCircuitBreakerPolicy(message, serviceOptions, circuitBreakerDuration,
+                    circuitBreakerSamplingDuration, circuitBreakerFailureThreshold, circuitBreakerMinimumThroughput,
+                    logger));
             return policy;
         });
     }
@@ -125,17 +123,21 @@ public static IHttpClientBuilder AddTimeoutPolicy(
     private static AsyncCircuitBreakerPolicy<HttpResponseMessage> BuildCircuitBreakerPolicy(
         HttpRequestMessage message,
         BaseServiceOptions serviceOptions,
-        int circuitBreakerExceptionsCount,
         TimeSpan circuitBreakerDuration,
+        TimeSpan circuitBreakerSamplingDuration,
+        double circuitBreakerFailureThreshold,
+        int circuitBreakerMinimumThroughput,
         ILogger logger)
     {
         return HttpPolicyExtensions
             .HandleTransientHttpError()
             .Or<TimeoutRejectedException>()
             .OrResult(r => r.StatusCode == (HttpStatusCode) 429) // Too Many Requests
-            .CircuitBreakerAsync(
-                circuitBreakerExceptionsCount,
-                circuitBreakerDuration,
+            .AdvancedCircuitBreakerAsync(
+                failureThreshold: circuitBreakerFailureThreshold,
+                samplingDuration: circuitBreakerSamplingDuration,
+                minimumThroughput: circuitBreakerMinimumThroughput,
+                durationOfBreak: circuitBreakerDuration,
                 (response, circuitState, timeSpan, _) =>
                 {
                     logger.ErrorWithObject(null, "CB onBreak", new

diff --git a/ATI.Services.Common/Options/BaseServiceOptions.cs b/ATI.Services.Common/Options/BaseServiceOptions.cs
@@ -38,15 +38,30 @@ public string ServiceName
     public TimeSpan MedianFirstRetryDelay { get; set; } = TimeSpan.FromSeconds(1);
 
     /// <summary>
-    /// Number of exceptions after which CB will be opened (will stop making requests)
-    /// Set 0 if you dont want to use CB
+    /// Enable/disable CB (enabled by default)
     /// </summary>
-    public int CircuitBreakerExceptionsCount { get; set; } = 20;
+    public bool CircuitBreakerEnabled = true;
 
     /// <summary>
     /// Time after which CB will be closed (will make requests)
     /// </summary>
     public TimeSpan CircuitBreakerDuration { get; set; } = TimeSpan.FromSeconds(2);
+
+    /// <summary>
+    /// Time window over which failure-success ratio calculated 
+    /// </summary>
+    public TimeSpan CircuitBreakerSamplingDuration { get; set; } = TimeSpan.FromSeconds(5);
+
+    /// <summary>
+    /// Failure-success ratio for circuit open
+    /// </summary>
+    public double CircuitBreakerFailureThreshold { get; set; } = 0.5;
+
+    /// <summary>
+    /// Minimal number of actions must occur for specified sampling duration that actions was significant for statistic
+    /// set 0 if don't want use CB
+    /// </summary>
+    public int CircuitBreakerMinimumThroughput { get; set; } = 10;
 
     public string Environment { get; set; }
     public TimeSpan? LongRequestTime { get; set; }

diff --git a/ATI.Services.Common/Policies/RetryPolicySettings.cs b/ATI.Services.Common/Policies/RetryPolicySettings.cs
@@ -16,14 +16,4 @@ public class RetryPolicySettings
     /// Median for spreading queries over time
     /// </summary>
     public TimeSpan? MedianFirstRetryDelay { get; set; }
-
-    /// <summary>
-    /// Number of exceptions after which CB will be opened (will stop making requests)
-    /// </summary>
-    public int? CircuitBreakerExceptionsCount { get; set; }
-
-    /// <summary>
-    /// Time after which CB will be closed (will make requests)
-    /// </summary>
-    public TimeSpan? CircuitBreakerDuration { get; set; }
 }
diff --git a/README.md b/README.md
@@ -225,26 +225,44 @@ public class FirmsAdapter
     /// Median for spreading queries over time
     /// </summary>
     public TimeSpan MedianFirstRetryDelay { get; set; } = TimeSpan.FromSeconds(1);
+
+    /// <summary>
+    /// Http methods to retry
+    /// If not set - retry only GET methods
+    /// </summary>
+    public List<string> HttpMethodsToRetry { get; set; }
+}
+```
 
+#### CircuitBreaker Settings in BaseServiceOptions
+```csharp
     /// <summary>
-    /// Number of exceptions after which CB will be opened (will stop making requests)
-    /// Set 0 if you dont want to use CB
+    /// Enable/disable CB (enabled by default)
     /// </summary>
-    public int CircuitBreakerExceptionsCount { get; set; } = 20;
-    
+    public bool CircuitBreakerEnabled = true;
+
     /// <summary>
     /// Time after which CB will be closed (will make requests)
     /// </summary>
     public TimeSpan CircuitBreakerDuration { get; set; } = TimeSpan.FromSeconds(2);
+
+    /// <summary>
+    /// Time window over which failure-success ratio calculated 
+    /// </summary>
+    public TimeSpan CircuitBreakerSamplingDuration { get; set; } = TimeSpan.FromSeconds(5);
+
+    /// <summary>
+    /// Failure-success ratio for circuit open
+    /// </summary>
+    public double CircuitBreakerFailureThreshold { get; set; } = 0.5;
 
     /// <summary>
-    /// Http methods to retry
-    /// If not set - retry only GET methods
+    /// Minimal number of actions must occur for specified sampling duration that actions was significant for statistic
     /// </summary>
-    public List<string> HttpMethodsToRetry { get; set; }
-}
+    public int CircuitBreakerMinimumThroughput { get; set; } = 10;
 ```
-Если вы вызовете `AddCustomHttpClient<>`, по умолчанию будут включены все Policies. Если вы хотите выключить RetryPolicy - поставьте значение 0 у параметра `RetryCount`, CB Policy - `CircuitBreakerExceptionsCount`.
+
+Если вы вызовете `AddCustomHttpClient<>`, по умолчанию будут включены все Policies. Если вы хотите выключить RetryPolicy - поставьте значение 0 у параметра `RetryCount`, CB Policy - `CircuitBreakerEnabled` : false.
 Также можно переопределить политики выполнения конкретного запроса. Для этого в `HttpClient.SendAsync` нужно передать настройку `retryPolicySettings`. NOTE - если передать не NULL, тогда проверки на `TServiceOptions.HttpMethodsToRetry` не будет (так как считаем, что `retryPolicySettings` имеет бОльший вес; но проверка на `RetryCount` - останется)
 
 ---