From a8b244b738aec69dc7f1f582a96ff321989084f4 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Mon, 26 Aug 2024 15:36:22 +0200 Subject: [PATCH 1/4] Add custom log source support --- collector.yaml | 27 +++++++++++++++++++ .../logsagentexporter/logs_exporter.go | 6 ++++- .../logsagentexporter/logs_exporter_test.go | 6 +++-- ...kjmr-otlp-log-source-f9bc51dcd803a238.yaml | 11 ++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 collector.yaml create mode 100644 releasenotes/notes/mackjmr-otlp-log-source-f9bc51dcd803a238.yaml diff --git a/collector.yaml b/collector.yaml new file mode 100644 index 0000000000000..f260c86ff4221 --- /dev/null +++ b/collector.yaml @@ -0,0 +1,27 @@ +receivers: + otlp: + protocols: + http: + endpoint: "localhost:4318" + grpc: + endpoint: "localhost:4317" + + +processors: + batch: + send_batch_max_size: 1000 + send_batch_size: 100 + timeout: 10s + +exporters: + datadog: + api: + site: ${env:DD_SITE} + key: ${env:DD_API_KEY} + +service: + pipelines: + logs: + receivers: [otlp] + processors: [batch] + exporters: [datadog] \ No newline at end of file diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter.go b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter.go index 08670311f98f4..7bf6c54154f23 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter.go +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter.go @@ -80,7 +80,11 @@ func (e *Exporter) ConsumeLogs(ctx context.Context, ld plog.Logs) (err error) { origin := message.NewOrigin(e.logSource) origin.SetTags(tags) origin.SetService(service) - origin.SetSource(e.logSource.Name) + if src, ok := ddLog.AdditionalProperties["datadog.log.source"]; ok { + origin.SetSource(src) + } else { + origin.SetSource(e.logSource.Name) + } content, err := ddLog.MarshalJSON() if err != nil { diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go index e7ef686b993f3..1a87e9f22991e 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go @@ -72,10 +72,11 @@ func TestLogsExporter(t *testing.T) { lrr := testutil.GenerateLogsOneLogRecord() ldd := lrr.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0) ldd.Attributes().PutStr("message", "hello") + ldd.Attributes().PutStr("datadog.log.source", "custom_source") return lrr }(), otelSource: otelSource, - logSourceName: LogSourceName, + logSourceName: "custom_source", }, want: testutil.JSONLogs{ @@ -83,6 +84,7 @@ func TestLogsExporter(t *testing.T) { "message": "hello", "app": "server", "instance_num": "1", + "datadog.log.source": "custom_source", "@timestamp": testutil.TestLogTime.Format("2006-01-02T15:04:05.000Z07:00"), "status": "Info", "dd.span_id": fmt.Sprintf("%d", spanIDToUint64(ld.SpanID())), @@ -186,7 +188,7 @@ func TestLogsExporter(t *testing.T) { return lrr }(), otelSource: "datadog_exporter", - logSourceName: "custom_source", + logSourceName: "", }, want: testutil.JSONLogs{ diff --git a/releasenotes/notes/mackjmr-otlp-log-source-f9bc51dcd803a238.yaml b/releasenotes/notes/mackjmr-otlp-log-source-f9bc51dcd803a238.yaml new file mode 100644 index 0000000000000..eb44689e78451 --- /dev/null +++ b/releasenotes/notes/mackjmr-otlp-log-source-f9bc51dcd803a238.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Add support for setting a custom log source from resource attribute or log attribute `datadog.log.source`. From 6a6c4536cdc46f1fb577bf46ee0573672574f8d6 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Mon, 26 Aug 2024 15:37:22 +0200 Subject: [PATCH 2/4] remove yaml --- collector.yaml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 collector.yaml diff --git a/collector.yaml b/collector.yaml deleted file mode 100644 index f260c86ff4221..0000000000000 --- a/collector.yaml +++ /dev/null @@ -1,27 +0,0 @@ -receivers: - otlp: - protocols: - http: - endpoint: "localhost:4318" - grpc: - endpoint: "localhost:4317" - - -processors: - batch: - send_batch_max_size: 1000 - send_batch_size: 100 - timeout: 10s - -exporters: - datadog: - api: - site: ${env:DD_SITE} - key: ${env:DD_API_KEY} - -service: - pipelines: - logs: - receivers: [otlp] - processors: [batch] - exporters: [datadog] \ No newline at end of file From fe16b1d2d98595695e79f269fb1db68198771544 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Fri, 30 Aug 2024 14:57:31 +0200 Subject: [PATCH 3/4] add resource attribute test --- .../logsagentexporter/logs_exporter_test.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go index 1a87e9f22991e..c7c459be4bda3 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go @@ -99,6 +99,40 @@ func TestLogsExporter(t *testing.T) { }, expectedTags: [][]string{{"otel_source:datadog_agent"}}, }, + { + name: "resource-attribute-source", + args: args{ + ld: func() plog.Logs { + l := testutil.GenerateLogsOneLogRecord() + rl := l.ResourceLogs().At(0) + resourceAttrs := rl.Resource().Attributes() + resourceAttrs.PutStr("datadog.log.source", "custom_source_rattr") + return l + }(), + otelSource: otelSource, + logSourceName: "custom_source_rattr", + }, + + want: testutil.JSONLogs{ + { + "message": "This is a log message", + "app": "server", + "instance_num": "1", + "datadog.log.source": "custom_source_rattr", + "@timestamp": testutil.TestLogTime.Format("2006-01-02T15:04:05.000Z07:00"), + "status": "Info", + "dd.span_id": fmt.Sprintf("%d", spanIDToUint64(ld.SpanID())), + "dd.trace_id": fmt.Sprintf("%d", traceIDToUint64(ld.TraceID())), + "otel.severity_text": "Info", + "otel.severity_number": "9", + "otel.span_id": spanIDToHexOrEmptyString(ld.SpanID()), + "otel.trace_id": traceIDToHexOrEmptyString(ld.TraceID()), + "otel.timestamp": fmt.Sprintf("%d", testutil.TestLogTime.UnixNano()), + "resource-attr": "resource-attr-val-1", + }, + }, + expectedTags: [][]string{{"otel_source:datadog_agent"}}, + }, { name: "ddtags", args: args{ From 89f9d4b7009d483c002f842e0aa9dc8557cb31c8 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Fri, 30 Aug 2024 16:08:09 +0200 Subject: [PATCH 4/4] address feedback --- .../exporter/logsagentexporter/logs_exporter_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go index c7c459be4bda3..3944c650f7828 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/logs_exporter_test.go @@ -76,7 +76,7 @@ func TestLogsExporter(t *testing.T) { return lrr }(), otelSource: otelSource, - logSourceName: "custom_source", + logSourceName: LogSourceName, }, want: testutil.JSONLogs{ @@ -110,7 +110,7 @@ func TestLogsExporter(t *testing.T) { return l }(), otelSource: otelSource, - logSourceName: "custom_source_rattr", + logSourceName: LogSourceName, }, want: testutil.JSONLogs{ @@ -277,7 +277,11 @@ func TestLogsExporter(t *testing.T) { output := <-testChannel outputJSON := make(map[string]interface{}) json.Unmarshal(output.GetContent(), &outputJSON) - assert.Equal(t, tt.args.logSourceName, output.Origin.Source()) + if src, ok := outputJSON["datadog.log.source"]; ok { + assert.Equal(t, src, output.Origin.Source()) + } else { + assert.Equal(t, tt.args.logSourceName, output.Origin.Source()) + } assert.Equal(t, tt.expectedTags[i], output.Origin.Tags(nil)) ans = append(ans, outputJSON) }