Support content headers in HttpClient logging

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Benchmarks/ErasingRedactorProvider.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Extensions.Compliance.Classification;
+using Microsoft.Extensions.Compliance.Redaction;
+
+namespace Microsoft.Extensions.Http.Diagnostics.Bench.Benchmarks;
+
+internal sealed class ErasingRedactorProvider : IRedactorProvider
+{
+    public static ErasingRedactorProvider Instance { get; } = new();
+
+    public Redactor GetRedactor(DataClassificationSet classifications) => ErasingRedactor.Instance;
+}

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Benchmarks/HeadersReaderBenchmark.cs

@@ -0,0 +1,107 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Net.Http;
+using BenchmarkDotNet.Attributes;
+using Microsoft.Extensions.Compliance.Testing;
+using Microsoft.Extensions.Http.Logging;
+using Microsoft.Extensions.Http.Logging.Internal;
+using Microsoft.Extensions.Telemetry.Internal;
+
+namespace Microsoft.Extensions.Http.Diagnostics.Bench.Benchmarks;
+
+public class HeadersReaderBenchmark
+{
+    private List<KeyValuePair<string, string>> _outputBuffer = null!;
+    private HttpHeadersReader _headersReader = null!;
+
+    [Params(0, 5, 15)]
+    public int HeadersCount { get; set; }
+
+    // This one can't be 0, because in that case HttpRequestReader simply doesn't call HttpHeadersReader
+    [Params(1, 3, 5, 10)]
+    public int HeadersToLogCount { get; set; }
+
+    [Params(false, true)]
+    public bool LogContentHeaders { get; set; }
+
+    public HttpRequestMessage? Request { get; set; }
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        _outputBuffer = new(capacity: 10240);
+        Request = new HttpRequestMessage(HttpMethod.Post, "https://www.microsoft.com");
+        for (var i = 0; i < HeadersCount; i++)
+        {
+            Request.Headers.Add($"Header{i}", $"Value{i}");
+        }
+
+        var options = new LoggingOptions { LogContentHeaders = LogContentHeaders };
+        for (var i = 0; i < HeadersToLogCount; i++)
+        {
+            options.RequestHeadersDataClasses.Add($"Header{i}", FakeTaxonomy.PublicData);
+        }
+
+        var redactor = new HttpHeadersRedactor(ErasingRedactorProvider.Instance);
+        _headersReader = new HttpHeadersReader(new StaticOptionsMonitor<LoggingOptions>(options), redactor);
+    }
+
+    [GlobalCleanup]
+    public void Cleanup()
+    {
+        Request?.Dispose();
+        _outputBuffer.Clear();
+        _outputBuffer = null!;
+    }
+
+    [Benchmark]
+    public void HeadersReader()
+    {
+        _headersReader.ReadRequestHeaders(Request!, _outputBuffer);
+    }
+}
+
+/*
+
+ These results show comparison between a plain logic (when we enumerate over "headersToLog" dictionary),
+ and an updated logic when we choose the strategy based on the number of headers to read and the number of headers to log.
+
+BenchmarkDotNet=v0.13.5, OS=Windows 11 (10.0.22631.2861), VM=Hyper-V
+Intel Xeon Platinum 8370C CPU 2.80GHz, 1 CPU, 16 logical and 8 physical cores
+.NET SDK=8.0.100
+  [Host] : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
+
+Job=MediumRun  Jit=RyuJit  Platform=X64
+Runtime=.NET 8.0  Server=True  Toolchain=InProcessEmitToolchain
+IterationCount=15  LaunchCount=2  WarmupCount=10
+
+|           Method | HeadersCount | HeadersToLogCount |         Mean |      Error |     StdDev |   Gen0 | Allocated |
+|----------------- |------------- |------------------ |-------------:|-----------:|-----------:|-------:|----------:|
+| HeadersReaderNew |            0 |                 1 |     4.372 ns |  0.0120 ns |  0.0175 ns |      - |         - |
+| HeadersReaderNew |            0 |                 5 |     4.459 ns |  0.0639 ns |  0.0916 ns |      - |         - |
+| HeadersReaderNew |            0 |                 3 |     4.648 ns |  0.2073 ns |  0.2972 ns |      - |         - |
+| HeadersReaderNew |            0 |                10 |     4.995 ns |  0.4255 ns |  0.6369 ns |      - |         - |
+| HeadersReaderOld |            0 |                 1 |    18.607 ns |  0.0492 ns |  0.0689 ns |      - |         - |
+| HeadersReaderOld |            0 |                 3 |    41.269 ns |  0.0759 ns |  0.1088 ns |      - |         - |
+| HeadersReaderOld |            0 |                 5 |    63.525 ns |  0.1879 ns |  0.2572 ns |      - |         - |
+| HeadersReaderOld |            0 |                10 |   119.163 ns |  0.6748 ns |  0.9678 ns |      - |         - |
+| HeadersReaderOld |           15 |                 1 |   204.099 ns |  1.3834 ns |  1.8936 ns | 0.0100 |     256 B |
+| HeadersReaderOld |            5 |                 1 |   205.044 ns |  1.7938 ns |  2.4553 ns | 0.0095 |     256 B |
+| HeadersReaderNew |           15 |                 1 |   216.454 ns |  1.1861 ns |  1.6236 ns | 0.0100 |     256 B |
+| HeadersReaderNew |            5 |                 1 |   217.499 ns |  1.5149 ns |  2.0736 ns | 0.0100 |     256 B |
+| HeadersReaderOld |            5 |                 3 |   630.982 ns |  4.4363 ns |  5.9224 ns | 0.0305 |     768 B |
+| HeadersReaderOld |           15 |                 3 |   641.780 ns |  4.4820 ns |  6.1351 ns | 0.0305 |     768 B |
+| HeadersReaderNew |           15 |                 3 |   669.542 ns |  3.6006 ns |  4.9285 ns | 0.0305 |     768 B |
+| HeadersReaderNew |            5 |                 3 |   677.061 ns |  3.8858 ns |  5.3190 ns | 0.0305 |     768 B |
+| HeadersReaderNew |            5 |                10 |   981.974 ns |  9.8502 ns | 13.4831 ns | 0.0515 |    1336 B |
+| HeadersReaderOld |            5 |                 5 | 1,126.471 ns |  7.1361 ns |  9.7680 ns | 0.0496 |    1280 B |
+| HeadersReaderOld |           15 |                 5 | 1,134.430 ns |  6.2873 ns |  8.3934 ns | 0.0496 |    1280 B |
+| HeadersReaderNew |           15 |                 5 | 1,153.805 ns |  3.0554 ns |  4.1823 ns | 0.0496 |    1280 B |
+| HeadersReaderNew |            5 |                 5 | 1,164.717 ns |  9.8121 ns | 13.4310 ns | 0.0496 |    1280 B |
+| HeadersReaderOld |            5 |                10 | 1,413.431 ns |  7.9666 ns | 10.9048 ns | 0.0496 |    1280 B |
+| HeadersReaderOld |           15 |                10 | 2,489.613 ns | 19.0100 ns | 26.0211 ns | 0.0992 |    2560 B |
+| HeadersReaderNew |           15 |                10 | 2,507.523 ns | 10.2106 ns | 13.9763 ns | 0.0992 |    2560 B |
+
+ */

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Benchmarks/HugeHttpCLientLoggingBenchmark.cs

@@ -16,28 +16,28 @@ public class HugeHttpClientLoggingBenchmark
     private const int ReadSizeLimit = 53248;
     private static HttpRequestMessage Request => new(HttpMethod.Post, "https://www.microsoft.com");
 
-    private static readonly System.Net.Http.HttpClient _hugeNoLog
+    private static readonly HttpClient _hugeNoLog
         = HttpClientFactory.CreateWithoutLogging(DataFileName);
 
-    private static readonly System.Net.Http.HttpClient _hugeLogAll
+    private static readonly HttpClient _hugeLogAll
         = HttpClientFactory.CreateWithLoggingLogAll(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _hugeLogRequest
+    private static readonly HttpClient _hugeLogRequest
         = HttpClientFactory.CreateWithLoggingLogRequest(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _hugeLogResponse
+    private static readonly HttpClient _hugeLogResponse
         = HttpClientFactory.CreateWithLoggingLogResponse(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _hugeNoLogChunked
+    private static readonly HttpClient _hugeNoLogChunked
         = HttpClientFactory.CreateWithoutLogging_ChunkedEncoding(DataFileName);
 
-    private static readonly System.Net.Http.HttpClient _hugeLogAllChunked
+    private static readonly HttpClient _hugeLogAllChunked
         = HttpClientFactory.CreateWithLoggingLogAll_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _hugeLogRequestChunked
+    private static readonly HttpClient _hugeLogRequestChunked
         = HttpClientFactory.CreateWithLoggingLogRequest_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _hugeLogResponseChunked
+    private static readonly HttpClient _hugeLogResponseChunked
         = HttpClientFactory.CreateWithLoggingLogResponse_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
     [Benchmark(Baseline = true)]

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Benchmarks/MediumHttpClientLoggingBenchmark.cs

@@ -16,28 +16,28 @@ public class MediumHttpClientLoggingBenchmark
     private const int ReadSizeLimit = 16384;
     private static HttpRequestMessage Request => new(HttpMethod.Post, "https://www.microsoft.com");
 
-    private static readonly System.Net.Http.HttpClient _mediumNoLog
+    private static readonly HttpClient _mediumNoLog
         = HttpClientFactory.CreateWithoutLogging(DataFileName);
 
-    private static readonly System.Net.Http.HttpClient _mediumLogAll
+    private static readonly HttpClient _mediumLogAll
         = HttpClientFactory.CreateWithLoggingLogAll(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _mediumLogRequest
+    private static readonly HttpClient _mediumLogRequest
         = HttpClientFactory.CreateWithLoggingLogRequest(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _mediumLogResponse
+    private static readonly HttpClient _mediumLogResponse
         = HttpClientFactory.CreateWithLoggingLogResponse(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _mediumNoLogChunked
+    private static readonly HttpClient _mediumNoLogChunked
         = HttpClientFactory.CreateWithoutLogging_ChunkedEncoding(DataFileName);
 
-    private static readonly System.Net.Http.HttpClient _mediumLogAllChunked
+    private static readonly HttpClient _mediumLogAllChunked
         = HttpClientFactory.CreateWithLoggingLogAll_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _mediumLogRequestChunked
+    private static readonly HttpClient _mediumLogRequestChunked
         = HttpClientFactory.CreateWithLoggingLogRequest_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _mediumLogResponseChunked
+    private static readonly HttpClient _mediumLogResponseChunked
         = HttpClientFactory.CreateWithLoggingLogResponse_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
     [Benchmark(Baseline = true)]

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Benchmarks/SmallHttpClientLoggingBenchmark.cs

@@ -16,28 +16,28 @@ public class SmallHttpClientLoggingBenchmark
     private const int ReadSizeLimit = 8192;
     private static HttpRequestMessage Request => new(HttpMethod.Post, "https://www.microsoft.com");
 
-    private static readonly System.Net.Http.HttpClient _smallNoLog
+    private static readonly HttpClient _smallNoLog
         = HttpClientFactory.CreateWithoutLogging(DataFileName);
 
-    private static readonly System.Net.Http.HttpClient _smallLogAll
+    private static readonly HttpClient _smallLogAll
         = HttpClientFactory.CreateWithLoggingLogAll(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _smallLogRequest
+    private static readonly HttpClient _smallLogRequest
         = HttpClientFactory.CreateWithLoggingLogRequest(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _smallLogResponse
+    private static readonly HttpClient _smallLogResponse
         = HttpClientFactory.CreateWithLoggingLogResponse(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _smallNoLogChunked
+    private static readonly HttpClient _smallNoLogChunked
         = HttpClientFactory.CreateWithoutLogging_ChunkedEncoding(DataFileName);
 
-    private static readonly System.Net.Http.HttpClient _smallLogAllChunked
+    private static readonly HttpClient _smallLogAllChunked
         = HttpClientFactory.CreateWithLoggingLogAll_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _smallLogRequestChunked
+    private static readonly HttpClient _smallLogRequestChunked
         = HttpClientFactory.CreateWithLoggingLogRequest_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
-    private static readonly System.Net.Http.HttpClient _smallLogResponseChunked
+    private static readonly HttpClient _smallLogResponseChunked
         = HttpClientFactory.CreateWithLoggingLogResponse_ChunkedEncoding(DataFileName, ReadSizeLimit);
 
     [Benchmark(Baseline = true)]

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/HttpClientFactory.cs

@@ -11,7 +11,7 @@ namespace Microsoft.Extensions.Http.Logging.Bench;
 
 internal static class HttpClientFactory
 {
-    public static System.Net.Http.HttpClient CreateWithLoggingLogRequest(string fileName, int readLimit)
+    public static HttpClient CreateWithLoggingLogRequest(string fileName, int readLimit)
     {
         var services = new ServiceCollection();
 
@@ -25,7 +25,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogRequest(string file
             .AddExtendedHttpClientLogging(options =>
             {
                 options.BodySizeLimit = readLimit;
-                options.RequestBodyContentTypes.Add(new("application/json"));
+                options.RequestBodyContentTypes.Add("application/json");
                 options.RequestHeadersDataClasses.Add("Content-Type", FakeTaxonomy.PrivateData);
             })
             .AddHttpMessageHandler<NoRemoteCallHandler>()
@@ -35,7 +35,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogRequest(string file
             .CreateClient(nameof(fileName));
     }
 
-    public static System.Net.Http.HttpClient CreateWithLoggingLogResponse(string fileName, int readLimit)
+    public static HttpClient CreateWithLoggingLogResponse(string fileName, int readLimit)
     {
         var services = new ServiceCollection();
 
@@ -48,7 +48,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogResponse(string fil
             .AddExtendedHttpClientLogging(options =>
             {
                 options.BodySizeLimit = readLimit;
-                options.ResponseBodyContentTypes.Add(new("application/json"));
+                options.ResponseBodyContentTypes.Add("application/json");
                 options.ResponseHeadersDataClasses.Add("Content-Type", FakeTaxonomy.PrivateData);
             })
             .AddHttpMessageHandler<NoRemoteCallHandler>()
@@ -58,7 +58,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogResponse(string fil
             .CreateClient(nameof(fileName));
     }
 
-    public static System.Net.Http.HttpClient CreateWithLoggingLogAll(string fileName, int readLimit)
+    public static HttpClient CreateWithLoggingLogAll(string fileName, int readLimit)
     {
         var services = new ServiceCollection();
 
@@ -72,10 +72,10 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogAll(string fileName
             {
                 options.BodySizeLimit = readLimit;
 
-                options.RequestBodyContentTypes.Add(new("application/json"));
+                options.RequestBodyContentTypes.Add("application/json");
                 options.RequestHeadersDataClasses.Add("Content-Type", FakeTaxonomy.PrivateData);
 
-                options.ResponseBodyContentTypes.Add(new("application/json"));
+                options.ResponseBodyContentTypes.Add("application/json");
                 options.ResponseHeadersDataClasses.Add("Content-Type", FakeTaxonomy.PrivateData);
             })
             .AddHttpMessageHandler<NoRemoteCallHandler>()
@@ -85,7 +85,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogAll(string fileName
             .CreateClient(nameof(fileName));
     }
 
-    public static System.Net.Http.HttpClient CreateWithLoggingLogRequest_ChunkedEncoding(string fileName, int readLimit)
+    public static HttpClient CreateWithLoggingLogRequest_ChunkedEncoding(string fileName, int readLimit)
     {
         var services = new ServiceCollection();
 
@@ -108,7 +108,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogRequest_ChunkedEnco
             .CreateClient(nameof(fileName));
     }
 
-    public static System.Net.Http.HttpClient CreateWithLoggingLogResponse_ChunkedEncoding(string fileName, int readLimit)
+    public static HttpClient CreateWithLoggingLogResponse_ChunkedEncoding(string fileName, int readLimit)
     {
         var services = new ServiceCollection();
 
@@ -131,7 +131,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogResponse_ChunkedEnc
             .CreateClient(nameof(fileName));
     }
 
-    public static System.Net.Http.HttpClient CreateWithLoggingLogAll_ChunkedEncoding(string fileName, int readLimit)
+    public static HttpClient CreateWithLoggingLogAll_ChunkedEncoding(string fileName, int readLimit)
     {
         var services = new ServiceCollection();
 
@@ -158,7 +158,7 @@ public static System.Net.Http.HttpClient CreateWithLoggingLogAll_ChunkedEncoding
             .CreateClient(nameof(fileName));
     }
 
-    public static System.Net.Http.HttpClient CreateWithoutLogging(string fileName)
+    public static HttpClient CreateWithoutLogging(string fileName)
         => new ServiceCollection()
             .AddSingleton(_ => NoRemoteCallHandler.Create(fileName))
             .AddHttpClient(nameof(fileName))
@@ -168,7 +168,7 @@ public static System.Net.Http.HttpClient CreateWithoutLogging(string fileName)
             .GetRequiredService<IHttpClientFactory>()
             .CreateClient(nameof(fileName));
 
-    public static System.Net.Http.HttpClient CreateWithoutLogging_ChunkedEncoding(string fileName)
+    public static HttpClient CreateWithoutLogging_ChunkedEncoding(string fileName)
         => new ServiceCollection()
             .AddSingleton(_ => NoRemoteCallNotSeekableHandler.Create(fileName))
             .AddHttpClient(nameof(fileName))

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Microsoft.Extensions.Http.Diagnostics.PerformanceTests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <RootNamespace>Microsoft.Extensions.Http.Diagnostics.Bench</RootNamespace>
     <Description>Benchmarks for Microsoft.Extensions.Http.Diagnostics.</Description>
@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\src\Libraries\Microsoft.Extensions.Http.Diagnostics\Microsoft.Extensions.Http.Diagnostics.csproj" />
+    <ProjectReference Include="..\..\..\src\Libraries\Microsoft.Extensions.Compliance.Redaction\Microsoft.Extensions.Compliance.Redaction.csproj" />
     <ProjectReference Include="..\..\..\src\Libraries\Microsoft.Extensions.Compliance.Testing\Microsoft.Extensions.Compliance.Testing.csproj" />
   </ItemGroup>
 </Project>

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/NoRemoteCallHandler.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
-using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Net.Http;
 using System.Threading;
@@ -19,8 +18,6 @@ private NoRemoteCallHandler(byte[] data)
         _data = data;
     }
 
-    [SuppressMessage("Performance Analysis", "CPR120:File.ReadAllXXX should be replaced by using a StreamReader to avoid adding objects to the large object heap (LOH).",
-        Justification = "We can live with it here")]
     public static NoRemoteCallHandler Create(string fileName)
     {
         var assemblyFileLocation = Path.GetDirectoryName(typeof(NoRemoteCallHandler).Assembly.Location)!;
@@ -41,7 +38,7 @@ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage reques
         {
             StatusCode = System.Net.HttpStatusCode.OK,
             RequestMessage = request,
-            Content = new StreamContent(new MemoryStream(_data))
+            Content = new StreamContent(new MemoryStream(_data, writable: false))
         };
 
         response.Content.Headers.ContentType = new("application/json");

bench/Libraries/Microsoft.Extensions.Http.Diagnostics.PerformanceTests/Program.cs

@@ -18,7 +18,7 @@ private static void Main(string[] args)
         var dontRequireSlnToRunBenchmarks = ManualConfig
             .Create(DefaultConfig.Instance)
             .AddJob(Job.MediumRun
-                .WithRuntime(CoreRuntime.Core50)
+                .WithRuntime(CoreRuntime.Core80)
                 .WithGcServer(true)
                 .WithJit(Jit.RyuJit)
                 .WithPlatform(Platform.X64)