diff --git a/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs b/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs
index 7c1b55df..fcaba42a 100644
--- a/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs
+++ b/src/Elastic.CommonSchema.Serilog/EcsTextFormatterConfiguration.cs
@@ -8,6 +8,7 @@
using System.Web;
#endif
using System;
+using System.Collections.Generic;
using System.Runtime.CompilerServices;
using Serilog.Events;
@@ -19,6 +20,7 @@ public interface IEcsTextFormatterConfiguration
Func MapCustom { get; set; }
bool MapExceptions { get; set; }
IHttpAdapter MapHttpAdapter { get; set; }
+ ISet LogEventPropertiesToFilter { get;set; }
}
public class EcsTextFormatterConfiguration : IEcsTextFormatterConfiguration
@@ -27,6 +29,7 @@ public class EcsTextFormatterConfiguration : IEcsTextFormatterConfiguration
bool IEcsTextFormatterConfiguration.MapCurrentThread { get; set; } = true;
IHttpAdapter IEcsTextFormatterConfiguration.MapHttpAdapter { get; set; }
+ ISet IEcsTextFormatterConfiguration.LogEventPropertiesToFilter { get; set; }
Func IEcsTextFormatterConfiguration.MapCustom { get; set; } = (b, e) => b;
@@ -43,6 +46,8 @@ public EcsTextFormatterConfiguration MapHttpContext(HttpContext httpContext) =>
public EcsTextFormatterConfiguration MapCustom(Func value) => Assign(this, value, (o, v) => o.MapCustom = v);
+ public EcsTextFormatterConfiguration LogEventPropertiesToFilter(ISet value) => Assign(this, value, (o, v) => o.LogEventPropertiesToFilter = v);
+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static EcsTextFormatterConfiguration Assign(
EcsTextFormatterConfiguration self, TValue value, Action assign
diff --git a/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs b/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs
index bd761961..b3952f1a 100644
--- a/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs
+++ b/src/Elastic.CommonSchema.Serilog/LogEventConverter.cs
@@ -56,7 +56,7 @@ public static Base ConvertToEcs(LogEvent logEvent, IEcsTextFormatterConfiguratio
Log = GetLog(logEvent, exceptions, configuration),
Agent = GetAgent(logEvent),
Event = GetEvent(logEvent),
- Metadata = GetMetadata(logEvent),
+ Metadata = GetMetadata(logEvent,configuration.LogEventPropertiesToFilter),
Process = GetProcess(logEvent, configuration.MapCurrentThread),
Host = GetHost(logEvent),
Trace = GetTrace(logEvent),
@@ -91,7 +91,7 @@ private static Transaction GetTransaction(LogEvent logEvent) =>
? null
: new Transaction { Id = transactionId.Value.ToString() };
- private static IDictionary GetMetadata(LogEvent logEvent)
+ private static IDictionary GetMetadata(LogEvent logEvent, ISet logEventPropertiesToFilter)
{
var dict = new Dictionary
{
@@ -137,7 +137,9 @@ private static IDictionary GetMetadata(LogEvent logEvent)
case SpecialKeys.MachineName:
continue;
}
-
+ //key present in list of keys to filter
+ if (logEventPropertiesToFilter?.Contains(logEventPropertyValue.Key) ?? false)
+ continue;
dict.Add(ToSnakeCase(logEventPropertyValue.Key), PropertyValueToObject(logEventPropertyValue.Value));
}
diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs
new file mode 100644
index 00000000..96a2d276
--- /dev/null
+++ b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs
@@ -0,0 +1,167 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Elastic.Apm;
+using Elastic.Apm.SerilogEnricher;
+using FluentAssertions;
+using Serilog;
+using Serilog.Events;
+using Serilog.Parsing;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Elastic.CommonSchema.Serilog.Tests
+{
+ public class LogEventPropFilterTests : LogTestsBase
+ {
+ public LogEventPropFilterTests(ITestOutputHelper output) : base(output)
+ {
+ LoggerConfiguration = LoggerConfiguration
+ .Enrich.WithThreadId()
+ .Enrich.WithThreadName()
+ .Enrich.WithMachineName()
+ .Enrich.WithProcessId()
+ .Enrich.WithProcessName()
+ .Enrich.WithEnvironmentUserName()
+ .Enrich.WithElasticApmCorrelationInfo();
+
+ Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+ .LogEventPropertiesToFilter(new HashSet(){{ "foo" }}));
+ }
+
+ private LogEvent BuildLogEvent()
+ {
+ var parser = new MessageTemplateParser();
+ return new LogEvent(
+ DateTimeOffset.Now,
+ LogEventLevel.Information,
+ null,
+ parser.Parse("My Log message!"),
+ new LogEventProperty[]
+ {
+ new LogEventProperty("foo", new ScalarValue("aaa")),
+ new LogEventProperty("bar", new ScalarValue("bbb")),
+ });
+ }
+
+ ///
+ /// Test the default via a hashset
+ ///
+ [Fact]
+ public void FilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+ {
+ var parser = new MessageTemplateParser();
+ var evnt = BuildLogEvent();
+ logger.Write(evnt);
+
+ var logEvents = getLogEvents();
+ logEvents.Should().HaveCount(1);
+
+ var ecsEvents = ToEcsEvents(logEvents);
+
+ var (_, info) = ecsEvents.First();
+ info.Log.Level.Should().Be("Information");
+ info.Error.Should().BeNull();
+ info.Metadata.Should().Contain("bar", "bbb");
+ info.Metadata.Should().NotContainKey("foo", "Should have been filtered");
+ });
+ ///
+ /// Test that null does not cause any critical errors
+ ///
+ [Fact]
+ public void NullFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+ {
+ Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+ .LogEventPropertiesToFilter(null));
+
+ var evnt = BuildLogEvent();
+ logger.Write(evnt);
+
+ var logEvents = getLogEvents();
+ logEvents.Should().HaveCount(1);
+
+ var ecsEvents = ToEcsEvents(logEvents);
+
+ var (_, info) = ecsEvents.First();
+ info.Log.Level.Should().Be("Information");
+ info.Error.Should().BeNull();
+ info.Metadata.Should().Contain("bar", "bbb");
+ info.Metadata.Should().Contain("foo", "aaa");
+ });
+
+ ///
+ /// Test that can be empty and does not cause any critical errors
+ ///
+ [Fact]
+ public void EmptyFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+ {
+ Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+ .LogEventPropertiesToFilter(new HashSet()));
+
+ var evnt = BuildLogEvent();
+ logger.Write(evnt);
+
+ var logEvents = getLogEvents();
+ logEvents.Should().HaveCount(1);
+
+ var ecsEvents = ToEcsEvents(logEvents);
+
+ var (_, info) = ecsEvents.First();
+ info.Log.Level.Should().Be("Information");
+ info.Error.Should().BeNull();
+ info.Metadata.Should().Contain("bar", "bbb");
+ info.Metadata.Should().Contain("foo", "aaa");
+ });
+ ///
+ /// Test that can be case insensitive
+ ///
+ [Fact]
+ public void CaseInsensitiveFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+ {
+ Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+ .LogEventPropertiesToFilter(new HashSet(StringComparer.OrdinalIgnoreCase){{ "FOO" }}));
+
+ var evnt = BuildLogEvent();
+ logger.Write(evnt);
+
+ var logEvents = getLogEvents();
+ logEvents.Should().HaveCount(1);
+
+ var ecsEvents = ToEcsEvents(logEvents);
+
+ var (_, info) = ecsEvents.First();
+ info.Log.Level.Should().Be("Information");
+ info.Error.Should().BeNull();
+ info.Metadata.Should().Contain("bar", "bbb");
+ info.Metadata.Should().NotContainKey("foo", "Should have been filtered");
+ });
+ ///
+ /// Test that can be case sensitive
+ ///
+ [Fact]
+ public void CaseSensitiveFilterLogEventProperty() => TestLogger((logger, getLogEvents) =>
+ {
+ Formatter = new EcsTextFormatter(new EcsTextFormatterConfiguration()
+ .LogEventPropertiesToFilter(new HashSet(StringComparer.Ordinal){{ "FOO" }}));
+
+ var evnt = BuildLogEvent();
+ logger.Write(evnt);
+
+ var logEvents = getLogEvents();
+ logEvents.Should().HaveCount(1);
+
+ var ecsEvents = ToEcsEvents(logEvents);
+
+ var (_, info) = ecsEvents.First();
+ info.Log.Level.Should().Be("Information");
+ info.Error.Should().BeNull();
+ info.Metadata.Should().Contain("bar", "bbb");
+ info.Metadata.Should().Contain("foo", "aaa");
+ });
+ }
+}
diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs b/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs
index c90b9b86..857dc6b0 100644
--- a/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs
+++ b/tests/Elastic.CommonSchema.Serilog.Tests/LogTestsBase.cs
@@ -17,7 +17,7 @@ public abstract class LogTestsBase
{
protected LoggerConfiguration LoggerConfiguration { get; set; }
- protected EcsTextFormatter Formatter { get; } = new EcsTextFormatter();
+ protected EcsTextFormatter Formatter { get; set; } = new EcsTextFormatter();
protected LogTestsBase(ITestOutputHelper output) =>
LoggerConfiguration = new LoggerConfiguration()