Feature/serilog property filter (#90)

This commit allows filtering of Serilog LogEvent properties from being added to EcsBase.Metadata

Co-authored-by: Russ Cam <[email protected]>

src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs

@@ -8,6 +8,7 @@
 using System.Web;
 #endif
 using System;
+using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using Serilog.Events;
 
@@ -19,6 +20,7 @@ public interface IEcsTextFormatterConfiguration
 		Func<Base, LogEvent, Base> MapCustom { get; set; }
 		bool MapExceptions { get; set; }
 		IHttpAdapter MapHttpAdapter { get; set; }
+		ISet<string> LogEventPropertiesToFilter { get;set; }
 	}
 
 	public class EcsTextFormatterConfiguration : IEcsTextFormatterConfiguration
@@ -27,6 +29,7 @@ public class EcsTextFormatterConfiguration : IEcsTextFormatterConfiguration
 		bool IEcsTextFormatterConfiguration.MapCurrentThread { get; set; } = true;
 
 		IHttpAdapter IEcsTextFormatterConfiguration.MapHttpAdapter { get; set; }
+		ISet<string> IEcsTextFormatterConfiguration.LogEventPropertiesToFilter { get; set; }
 
 		Func<Base, LogEvent, Base> IEcsTextFormatterConfiguration.MapCustom { get; set; } = (b, e) => b;
 
@@ -43,6 +46,8 @@ public EcsTextFormatterConfiguration MapHttpContext(HttpContext httpContext) =>
 
 		public EcsTextFormatterConfiguration MapCustom(Func<Base, LogEvent, Base> value) => Assign(this, value, (o, v) => o.MapCustom = v);
 
+		public EcsTextFormatterConfiguration LogEventPropertiesToFilter(ISet<string> value) => Assign(this, value, (o, v) => o.LogEventPropertiesToFilter = v);
+
 		[MethodImpl(MethodImplOptions.AggressiveInlining)]
 		private static EcsTextFormatterConfiguration Assign<TValue>(
 			EcsTextFormatterConfiguration self, TValue value, Action<IEcsTextFormatterConfiguration, TValue> assign

src/Elastic.CommonSchema.Serilog/LogEventConverter.cs

@@ -56,7 +56,7 @@ public static Base ConvertToEcs(LogEvent logEvent, IEcsTextFormatterConfiguratio
 				Log = GetLog(logEvent, exceptions, configuration),
 				Agent = GetAgent(logEvent),
 				Event = GetEvent(logEvent),
-				Metadata = GetMetadata(logEvent),
+				Metadata = GetMetadata(logEvent,configuration.LogEventPropertiesToFilter),
 				Process = GetProcess(logEvent, configuration.MapCurrentThread),
 				Host = GetHost(logEvent),
 				Trace = GetTrace(logEvent),
@@ -91,7 +91,7 @@ private static Transaction GetTransaction(LogEvent logEvent) =>
 				? null
 				: new Transaction { Id = transactionId.Value.ToString() };
 
-		private static IDictionary<string, object> GetMetadata(LogEvent logEvent)
+		private static IDictionary<string, object> GetMetadata(LogEvent logEvent, ISet<string> logEventPropertiesToFilter)
 		{
 			var dict = new Dictionary<string, object>
 			{
@@ -137,7 +137,9 @@ private static IDictionary<string, object> GetMetadata(LogEvent logEvent)
 					case SpecialKeys.MachineName:
 						continue;
 				}
-
+				//key present in list of keys to filter
+				if (logEventPropertiesToFilter?.Contains(logEventPropertyValue.Key) ?? false)
+					continue;
 				dict.Add(ToSnakeCase(logEventPropertyValue.Key), PropertyValueToObject(logEventPropertyValue.Value));
 			}
 

tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs

@@ -0,0 +1,167 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Elastic.Apm;
+using Elastic.Apm.SerilogEnricher;
+using FluentAssertions;
+using Serilog;
+using Serilog.Events;
+using Serilog.Parsing;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Elastic.CommonSchema.Serilog.Tests
+{
+	public class LogEventPropFilterTests : LogTestsBase
+	{
+		public LogEventPropFilterTests(ITestOutputHelper output) : base(output)
+		{
+			LoggerConfiguration = LoggerConfiguration
+				.Enrich.WithThreadId()
+				.Enrich.WithThreadName()
+				.Enrich.WithMachineName()
+				.Enrich.WithProcessId()
+				.Enrich.WithProcessName()
+				.Enrich.WithEnvironmentUserName()
+				.Enrich.WithElasticApmCorrelationInfo();
+
+			Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+					.LogEventPropertiesToFilter(new HashSet<string>(){{ "foo" }}));
+		}
+
+		private LogEvent BuildLogEvent()
+		{
+			var parser = new MessageTemplateParser();
+			return new LogEvent(
+				DateTimeOffset.Now,
+				LogEventLevel.Information,
+				null,
+				parser.Parse("My Log message!"),
+				new LogEventProperty[]
+				{
+					new LogEventProperty("foo", new ScalarValue("aaa")),
+					new LogEventProperty("bar", new ScalarValue("bbb")),
+				});
+		}
+
+		/// <summary>
+		/// Test the default <see cref="EcsTextFormatterConfiguration.LogEventPropertiesToFilter"/> via a hashset
+		/// </summary>
+		[Fact]
+		public void FilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+		{
+			var parser = new MessageTemplateParser();
+			var evnt = BuildLogEvent();
+			logger.Write(evnt);
+
+			var logEvents = getLogEvents();
+			logEvents.Should().HaveCount(1);
+
+			var ecsEvents = ToEcsEvents(logEvents);
+
+			var (_, info) = ecsEvents.First();
+			info.Log.Level.Should().Be("Information");
+			info.Error.Should().BeNull();
+			info.Metadata.Should().Contain("bar", "bbb");
+			info.Metadata.Should().NotContainKey("foo", "Should have been filtered");
+		});
+		/// <summary>
+		/// Test that null <see cref="EcsTextFormatterConfiguration.LogEventPropertiesToFilter"/> does not cause any critical errors
+		/// </summary>
+		[Fact]
+		public void NullFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+		{
+			Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+					.LogEventPropertiesToFilter(null));
+
+			var evnt = BuildLogEvent();
+			logger.Write(evnt);
+
+			var logEvents = getLogEvents();
+			logEvents.Should().HaveCount(1);
+
+			var ecsEvents = ToEcsEvents(logEvents);
+
+			var (_, info) = ecsEvents.First();
+			info.Log.Level.Should().Be("Information");
+			info.Error.Should().BeNull();
+			info.Metadata.Should().Contain("bar", "bbb");
+			info.Metadata.Should().Contain("foo", "aaa");
+		});
+
+		/// <summary>
+		/// Test that <see cref="EcsTextFormatterConfiguration.LogEventPropertiesToFilter"/> can be empty and does not cause any critical errors
+		/// </summary>
+		[Fact]
+		public void EmptyFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+		{
+			Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+				.LogEventPropertiesToFilter(new HashSet<string>()));
+
+			var evnt = BuildLogEvent();
+			logger.Write(evnt);
+
+			var logEvents = getLogEvents();
+			logEvents.Should().HaveCount(1);
+
+			var ecsEvents = ToEcsEvents(logEvents);
+
+			var (_, info) = ecsEvents.First();
+			info.Log.Level.Should().Be("Information");
+			info.Error.Should().BeNull();
+			info.Metadata.Should().Contain("bar", "bbb");
+			info.Metadata.Should().Contain("foo", "aaa");
+		});
+		/// <summary>
+		/// Test that <see cref="EcsTextFormatterConfiguration.LogEventPropertiesToFilter"/> can be case insensitive
+		/// </summary>
+		[Fact]
+		public void CaseInsensitiveFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+		{
+			Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+				.LogEventPropertiesToFilter(new HashSet<string>(StringComparer.OrdinalIgnoreCase){{ "FOO" }}));
+
+			var evnt = BuildLogEvent();
+			logger.Write(evnt);
+
+			var logEvents = getLogEvents();
+			logEvents.Should().HaveCount(1);
+
+			var ecsEvents = ToEcsEvents(logEvents);
+
+			var (_, info) = ecsEvents.First();
+			info.Log.Level.Should().Be("Information");
+			info.Error.Should().BeNull();
+			info.Metadata.Should().Contain("bar", "bbb");
+			info.Metadata.Should().NotContainKey("foo", "Should have been filtered");
+		});
+		/// <summary>
+		/// Test that <see cref="EcsTextFormatterConfiguration.LogEventPropertiesToFilter"/> can be case sensitive
+		/// </summary>
+		[Fact]
+		public void CaseSensitiveFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+		{
+			Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+				.LogEventPropertiesToFilter(new HashSet<string>(StringComparer.Ordinal){{ "FOO" }}));
+
+			var evnt = BuildLogEvent();
+			logger.Write(evnt);
+
+			var logEvents = getLogEvents();
+			logEvents.Should().HaveCount(1);
+
+			var ecsEvents = ToEcsEvents(logEvents);
+
+			var (_, info) = ecsEvents.First();
+			info.Log.Level.Should().Be("Information");
+			info.Error.Should().BeNull();
+			info.Metadata.Should().Contain("bar", "bbb");
+			info.Metadata.Should().Contain("foo", "aaa");
+		});
+	}
+}

tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs

@@ -17,7 +17,7 @@ public abstract class LogTestsBase
 	{
 		protected LoggerConfiguration LoggerConfiguration { get; set; }
 
-		protected EcsTextFormatter Formatter { get; } = new EcsTextFormatter();
+		protected EcsTextFormatter Formatter { get; set; } = new EcsTextFormatter();
 
 		protected LogTestsBase(ITestOutputHelper output) =>
 			LoggerConfiguration = new LoggerConfiguration()