diff --git a/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs b/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs index 7c1b55df..fcaba42a 100644 --- a/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs +++ b/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs @@ -8,6 +8,7 @@ using System.Web; #endif using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using Serilog.Events; @@ -19,6 +20,7 @@ public interface IEcsTextFormatterConfiguration Func MapCustom { get; set; } bool MapExceptions { get; set; } IHttpAdapter MapHttpAdapter { get; set; } + ISet LogEventPropertiesToFilter { get;set; } } public class EcsTextFormatterConfiguration : IEcsTextFormatterConfiguration @@ -27,6 +29,7 @@ public class EcsTextFormatterConfiguration : IEcsTextFormatterConfiguration bool IEcsTextFormatterConfiguration.MapCurrentThread { get; set; } = true; IHttpAdapter IEcsTextFormatterConfiguration.MapHttpAdapter { get; set; } + ISet IEcsTextFormatterConfiguration.LogEventPropertiesToFilter { get; set; } Func IEcsTextFormatterConfiguration.MapCustom { get; set; } = (b, e) => b; @@ -43,6 +46,8 @@ public EcsTextFormatterConfiguration MapHttpContext(HttpContext httpContext) => public EcsTextFormatterConfiguration MapCustom(Func value) => Assign(this, value, (o, v) => o.MapCustom = v); + public EcsTextFormatterConfiguration LogEventPropertiesToFilter(ISet value) => Assign(this, value, (o, v) => o.LogEventPropertiesToFilter = v); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static EcsTextFormatterConfiguration Assign( EcsTextFormatterConfiguration self, TValue value, Action assign diff --git a/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs b/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs index bd761961..b3952f1a 100644 --- a/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs +++ b/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs @@ -56,7 +56,7 @@ public static Base ConvertToEcs(LogEvent logEvent, IEcsTextFormatterConfiguratio Log = GetLog(logEvent, exceptions, configuration), Agent = GetAgent(logEvent), Event = GetEvent(logEvent), - Metadata = GetMetadata(logEvent), + Metadata = GetMetadata(logEvent,configuration.LogEventPropertiesToFilter), Process = GetProcess(logEvent, configuration.MapCurrentThread), Host = GetHost(logEvent), Trace = GetTrace(logEvent), @@ -91,7 +91,7 @@ private static Transaction GetTransaction(LogEvent logEvent) => ? null : new Transaction { Id = transactionId.Value.ToString() }; - private static IDictionary GetMetadata(LogEvent logEvent) + private static IDictionary GetMetadata(LogEvent logEvent, ISet logEventPropertiesToFilter) { var dict = new Dictionary { @@ -137,7 +137,9 @@ private static IDictionary GetMetadata(LogEvent logEvent) case SpecialKeys.MachineName: continue; } - + //key present in list of keys to filter + if (logEventPropertiesToFilter?.Contains(logEventPropertyValue.Key) ?? false) + continue; dict.Add(ToSnakeCase(logEventPropertyValue.Key), PropertyValueToObject(logEventPropertyValue.Value)); } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs new file mode 100644 index 00000000..96a2d276 --- /dev/null +++ b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs @@ -0,0 +1,167 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Elastic.Apm; +using Elastic.Apm.SerilogEnricher; +using FluentAssertions; +using Serilog; +using Serilog.Events; +using Serilog.Parsing; +using Xunit; +using Xunit.Abstractions; + +namespace Elastic.CommonSchema.Serilog.Tests +{ + public class LogEventPropFilterTests : LogTestsBase + { + public LogEventPropFilterTests(ITestOutputHelper output) : base(output) + { + LoggerConfiguration = LoggerConfiguration + .Enrich.WithThreadId() + .Enrich.WithThreadName() + .Enrich.WithMachineName() + .Enrich.WithProcessId() + .Enrich.WithProcessName() + .Enrich.WithEnvironmentUserName() + .Enrich.WithElasticApmCorrelationInfo(); + + Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration() + .LogEventPropertiesToFilter(new HashSet(){{ "foo" }})); + } + + private LogEvent BuildLogEvent() + { + var parser = new MessageTemplateParser(); + return new LogEvent( + DateTimeOffset.Now, + LogEventLevel.Information, + null, + parser.Parse("My Log message!"), + new LogEventProperty[] + { + new LogEventProperty("foo", new ScalarValue("aaa")), + new LogEventProperty("bar", new ScalarValue("bbb")), + }); + } + + /// + /// Test the default via a hashset + /// + [Fact] + public void FilterLogEventProperty() => TestLogger((logger, getLogEvents) => + { + var parser = new MessageTemplateParser(); + var evnt = BuildLogEvent(); + logger.Write(evnt); + + var logEvents = getLogEvents(); + logEvents.Should().HaveCount(1); + + var ecsEvents = ToEcsEvents(logEvents); + + var (_, info) = ecsEvents.First(); + info.Log.Level.Should().Be("Information"); + info.Error.Should().BeNull(); + info.Metadata.Should().Contain("bar", "bbb"); + info.Metadata.Should().NotContainKey("foo", "Should have been filtered"); + }); + /// + /// Test that null does not cause any critical errors + /// + [Fact] + public void NullFilterLogEventProperty() => TestLogger((logger, getLogEvents) => + { + Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration() + .LogEventPropertiesToFilter(null)); + + var evnt = BuildLogEvent(); + logger.Write(evnt); + + var logEvents = getLogEvents(); + logEvents.Should().HaveCount(1); + + var ecsEvents = ToEcsEvents(logEvents); + + var (_, info) = ecsEvents.First(); + info.Log.Level.Should().Be("Information"); + info.Error.Should().BeNull(); + info.Metadata.Should().Contain("bar", "bbb"); + info.Metadata.Should().Contain("foo", "aaa"); + }); + + /// + /// Test that can be empty and does not cause any critical errors + /// + [Fact] + public void EmptyFilterLogEventProperty() => TestLogger((logger, getLogEvents) => + { + Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration() + .LogEventPropertiesToFilter(new HashSet())); + + var evnt = BuildLogEvent(); + logger.Write(evnt); + + var logEvents = getLogEvents(); + logEvents.Should().HaveCount(1); + + var ecsEvents = ToEcsEvents(logEvents); + + var (_, info) = ecsEvents.First(); + info.Log.Level.Should().Be("Information"); + info.Error.Should().BeNull(); + info.Metadata.Should().Contain("bar", "bbb"); + info.Metadata.Should().Contain("foo", "aaa"); + }); + /// + /// Test that can be case insensitive + /// + [Fact] + public void CaseInsensitiveFilterLogEventProperty() => TestLogger((logger, getLogEvents) => + { + Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration() + .LogEventPropertiesToFilter(new HashSet(StringComparer.OrdinalIgnoreCase){{ "FOO" }})); + + var evnt = BuildLogEvent(); + logger.Write(evnt); + + var logEvents = getLogEvents(); + logEvents.Should().HaveCount(1); + + var ecsEvents = ToEcsEvents(logEvents); + + var (_, info) = ecsEvents.First(); + info.Log.Level.Should().Be("Information"); + info.Error.Should().BeNull(); + info.Metadata.Should().Contain("bar", "bbb"); + info.Metadata.Should().NotContainKey("foo", "Should have been filtered"); + }); + /// + /// Test that can be case sensitive + /// + [Fact] + public void CaseSensitiveFilterLogEventProperty() => TestLogger((logger, getLogEvents) => + { + Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration() + .LogEventPropertiesToFilter(new HashSet(StringComparer.Ordinal){{ "FOO" }})); + + var evnt = BuildLogEvent(); + logger.Write(evnt); + + var logEvents = getLogEvents(); + logEvents.Should().HaveCount(1); + + var ecsEvents = ToEcsEvents(logEvents); + + var (_, info) = ecsEvents.First(); + info.Log.Level.Should().Be("Information"); + info.Error.Should().BeNull(); + info.Metadata.Should().Contain("bar", "bbb"); + info.Metadata.Should().Contain("foo", "aaa"); + }); + } +} diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs b/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs index c90b9b86..857dc6b0 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs @@ -17,7 +17,7 @@ public abstract class LogTestsBase { protected LoggerConfiguration LoggerConfiguration { get; set; } - protected EcsTextFormatter Formatter { get; } = new EcsTextFormatter(); + protected EcsTextFormatter Formatter { get; set; } = new EcsTextFormatter(); protected LogTestsBase(ITestOutputHelper output) => LoggerConfiguration = new LoggerConfiguration()