diff --git a/.chloggen/exporter-elasticsarch-more-ecs-mode.yaml b/.chloggen/exporter-elasticsarch-more-ecs-mode.yaml new file mode 100644 index 000000000000..c5bfd6330225 --- /dev/null +++ b/.chloggen/exporter-elasticsarch-more-ecs-mode.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Converts more SemConv fields in OTel events to ECS fields in Elasticsearch documents when `mapping.mode: ecs` is specified." + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [31694] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 52490b1867e0..61cec87cd1f2 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -63,7 +63,7 @@ This exporter supports sending OpenTelemetry logs and traces to [Elasticsearch]( - `mode` (default=none): The fields naming mode. valid modes are: - `none`: Use original fields and event structure from the OTLP event. - `ecs`: Try to map fields defined in the - [OpenTelemetry Semantic Conventions](https://github.com/open-telemetry/semantic-conventions) + [OpenTelemetry Semantic Conventions](https://github.com/open-telemetry/semantic-conventions) (version 1.22.0) to [Elastic Common Schema (ECS)](https://www.elastic.co/guide/en/ecs/current/index.html). :warning: This mode's behavior is unstable, it is currently undergoing changes - `raw`: Omit the `Attributes.` string prefixed to field names for log and span attributes as well as omit the `Events.` string prefixed to diff --git a/exporter/elasticsearchexporter/logs_exporter_test.go b/exporter/elasticsearchexporter/logs_exporter_test.go index 2cf90b9b75c7..98d1720c2549 100644 --- a/exporter/elasticsearchexporter/logs_exporter_test.go +++ b/exporter/elasticsearchexporter/logs_exporter_test.go @@ -173,7 +173,7 @@ func TestExporter_PushEvent(t *testing.T) { server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { rec.Record(docs) - expected := `{"@timestamp":"1970-01-01T00:00:00.000000000Z","application":"myapp","attrKey1":"abc","attrKey2":"def","error":{"stack_trace":"no no no no"},"message":"hello world","service":{"name":"myservice"}}` + expected := `{"attrKey1":"abc","attrKey2":"def","application":"myapp","service":{"name":"myservice"},"error":{"stacktrace":"no no no no"},"agent":{"name":"otlp"},"@timestamp":"1970-01-01T00:00:00.000000000Z","message":"hello world"}` actual := string(docs[0].Document) assert.Equal(t, expected, actual) @@ -187,14 +187,14 @@ func TestExporter_PushEvent(t *testing.T) { mustSendLogsWithAttributes(t, exporter, // record attrs map[string]string{ - "application": "myapp", - "service.name": "myservice", + "application": "myapp", + "service.name": "myservice", + "exception.stacktrace": "no no no no", }, // resource attrs map[string]string{ - "attrKey1": "abc", - "attrKey2": "def", - "exception.stacktrace": "no no no no", + "attrKey1": "abc", + "attrKey2": "def", }, // record body "hello world", diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index cccdfa386923..b3e8829a3006 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -6,16 +6,49 @@ package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry import ( "bytes" "encoding/json" + "fmt" "time" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/ptrace" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter/internal/objmodel" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil" ) +// resourceAttrsConversionMap contains conversions for resource-level attributes +// from their Semantic Conventions (SemConv) names to equivalent Elastic Common +// Schema (ECS) names. +// If the ECS field name is specified as an empty string (""), the converter will +// neither convert the SemConv key to the equivalent ECS name nor pass-through the +// SemConv key as-is to become the ECS name. +var resourceAttrsConversionMap = map[string]string{ + semconv.AttributeServiceInstanceID: "service.node.name", + semconv.AttributeDeploymentEnvironment: "service.environment", + semconv.AttributeTelemetrySDKName: "", + semconv.AttributeTelemetrySDKLanguage: "", + semconv.AttributeTelemetrySDKVersion: "", + semconv.AttributeTelemetryDistroName: "", + semconv.AttributeTelemetryDistroVersion: "", + semconv.AttributeCloudPlatform: "cloud.service.name", + semconv.AttributeContainerImageTags: "container.image.tag", + semconv.AttributeHostName: "host.hostname", + semconv.AttributeHostArch: "host.architecture", + semconv.AttributeProcessExecutablePath: "process.executable", + semconv.AttributeProcessRuntimeName: "service.runtime.name", + semconv.AttributeProcessRuntimeVersion: "service.runtime.version", + semconv.AttributeOSName: "host.os.name", + semconv.AttributeOSType: "host.os.platform", + semconv.AttributeOSDescription: "host.os.full", + semconv.AttributeOSVersion: "host.os.version", + "k8s.namespace.name": "kubernetes.namespace", + "k8s.node.name": "kubernetes.node.name", + "k8s.pod.name": "kubernetes.pod.name", + "k8s.pod.uid": "kubernetes.pod.uid", +} + type mappingModel interface { encodeLog(pcommon.Resource, plog.LogRecord, pcommon.InstrumentationScope) ([]byte, error) encodeSpan(pcommon.Resource, ptrace.Span, pcommon.InstrumentationScope) ([]byte, error) @@ -41,72 +74,35 @@ const ( func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord, scope pcommon.InstrumentationScope) ([]byte, error) { var document objmodel.Document - switch m.mode { case MappingECS: - if record.Timestamp() != 0 { - document.AddTimestamp("@timestamp", record.Timestamp()) - } else { - document.AddTimestamp("@timestamp", record.ObservedTimestamp()) - } - - document.AddTraceID("trace.id", record.TraceID()) - document.AddSpanID("span.id", record.SpanID()) - - if n := record.SeverityNumber(); n != plog.SeverityNumberUnspecified { - document.AddInt("event.severity", int64(record.SeverityNumber())) - } - - document.AddString("log.level", record.SeverityText()) + document = m.encodeLogECSMode(resource, record, scope) + default: + document = m.encodeLogDefaultMode(resource, record, scope) + } - if record.Body().Type() == pcommon.ValueTypeStr { - document.AddAttribute("message", record.Body()) - } + var buf bytes.Buffer + err := document.Serialize(&buf, m.dedot) + return buf.Bytes(), err +} - fieldMapper := func(k string) string { - switch k { - case "exception.type": - return "error.type" - case "exception.message": - return "error.message" - case "exception.stacktrace": - return "error.stack_trace" - default: - return k - } - } +func (m *encodeModel) encodeLogDefaultMode(resource pcommon.Resource, record plog.LogRecord, scope pcommon.InstrumentationScope) objmodel.Document { + var document objmodel.Document - resource.Attributes().Range(func(k string, v pcommon.Value) bool { - k = fieldMapper(k) - document.AddAttribute(k, v) - return true - }) - scope.Attributes().Range(func(k string, v pcommon.Value) bool { - k = fieldMapper(k) - document.AddAttribute(k, v) - return true - }) - record.Attributes().Range(func(k string, v pcommon.Value) bool { - k = fieldMapper(k) - document.AddAttribute(k, v) - return true - }) - default: - docTimeStamp := record.Timestamp() - if docTimeStamp.AsTime().UnixNano() == 0 { - docTimeStamp = record.ObservedTimestamp() - } - document.AddTimestamp("@timestamp", docTimeStamp) // We use @timestamp in order to ensure that we can index if the default data stream logs template is used. - document.AddTraceID("TraceId", record.TraceID()) - document.AddSpanID("SpanId", record.SpanID()) - document.AddInt("TraceFlags", int64(record.Flags())) - document.AddString("SeverityText", record.SeverityText()) - document.AddInt("SeverityNumber", int64(record.SeverityNumber())) - document.AddAttribute("Body", record.Body()) - m.encodeAttributes(&document, record.Attributes()) - document.AddAttributes("Resource", resource.Attributes()) - document.AddAttributes("Scope", scopeToAttributes(scope)) + docTimeStamp := record.Timestamp() + if docTimeStamp.AsTime().UnixNano() == 0 { + docTimeStamp = record.ObservedTimestamp() } + document.AddTimestamp("@timestamp", docTimeStamp) // We use @timestamp in order to ensure that we can index if the default data stream logs template is used. + document.AddTraceID("TraceId", record.TraceID()) + document.AddSpanID("SpanId", record.SpanID()) + document.AddInt("TraceFlags", int64(record.Flags())) + document.AddString("SeverityText", record.SeverityText()) + document.AddInt("SeverityNumber", int64(record.SeverityNumber())) + document.AddAttribute("Body", record.Body()) + m.encodeAttributes(&document, record.Attributes()) + document.AddAttributes("Resource", resource.Attributes()) + document.AddAttributes("Scope", scopeToAttributes(scope)) if m.dedup { document.Dedup() @@ -114,9 +110,50 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord document.Sort() } - var buf bytes.Buffer - err := document.Serialize(&buf, m.dedot) - return buf.Bytes(), err + return document + +} + +func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.LogRecord, scope pcommon.InstrumentationScope) objmodel.Document { + var document objmodel.Document + + // First, try to map resource-level attributes to ECS fields. + encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap) + + // Then, try to map scope-level attributes to ECS fields. + scopeAttrsConversionMap := map[string]string{ + // None at the moment + } + encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap) + + // Finally, try to map record-level attributes to ECS fields. + recordAttrsConversionMap := map[string]string{ + "event.name": "event.action", + semconv.AttributeExceptionMessage: "error.message", + semconv.AttributeExceptionStacktrace: "error.stacktrace", + semconv.AttributeExceptionType: "error.type", + semconv.AttributeExceptionEscaped: "event.error.exception.handled", + } + encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap) + + // Handle special cases. + encodeLogAgentNameECSMode(&document, resource) + encodeLogAgentVersionECSMode(&document, resource) + encodeLogHostOsTypeECSMode(&document, resource) + encodeLogTimestampECSMode(&document, record) + document.AddTraceID("trace.id", record.TraceID()) + document.AddSpanID("span.id", record.SpanID()) + if n := record.SeverityNumber(); n != plog.SeverityNumberUnspecified { + document.AddInt("event.severity", int64(record.SeverityNumber())) + } + + document.AddString("log.level", record.SeverityText()) + + if record.Body().Type() == pcommon.ValueTypeStr { + document.AddAttribute("message", record.Body()) + } + + return document } func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, scope pcommon.InstrumentationScope) ([]byte, error) { @@ -193,3 +230,117 @@ func scopeToAttributes(scope pcommon.InstrumentationScope) pcommon.Map { } return attrs } + +func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string) { + if len(conversionMap) == 0 { + // No conversions to be done; add all attributes at top level of + // document. + document.AddAttributes("", attrs) + return + } + + attrs.Range(func(k string, v pcommon.Value) bool { + // If ECS key is found for current k in conversion map, use it. + if ecsKey, exists := conversionMap[k]; exists { + if ecsKey == "" { + // Skip the conversion for this k. + return true + } + + document.AddAttribute(ecsKey, v) + return true + } + + // Otherwise, add key at top level with attribute name as-is. + document.AddAttribute(k, v) + return true + }) +} + +func encodeLogAgentNameECSMode(document *objmodel.Document, resource pcommon.Resource) { + // Parse out telemetry SDK name, language, and distro name from resource + // attributes, setting defaults as needed. + telemetrySdkName := "otlp" + var telemetrySdkLanguage, telemetryDistroName string + + attrs := resource.Attributes() + if v, exists := attrs.Get(semconv.AttributeTelemetrySDKName); exists { + telemetrySdkName = v.Str() + } + if v, exists := attrs.Get(semconv.AttributeTelemetrySDKLanguage); exists { + telemetrySdkLanguage = v.Str() + } + if v, exists := attrs.Get(semconv.AttributeTelemetryDistroName); exists { + telemetryDistroName = v.Str() + if telemetrySdkLanguage == "" { + telemetrySdkLanguage = "unknown" + } + } + + // Construct agent name from telemetry SDK name, language, and distro name. + agentName := telemetrySdkName + if telemetryDistroName != "" { + agentName = fmt.Sprintf("%s/%s/%s", agentName, telemetrySdkLanguage, telemetryDistroName) + } else if telemetrySdkLanguage != "" { + agentName = fmt.Sprintf("%s/%s", agentName, telemetrySdkLanguage) + } + + // Set agent name in document. + document.AddString("agent.name", agentName) +} + +func encodeLogAgentVersionECSMode(document *objmodel.Document, resource pcommon.Resource) { + attrs := resource.Attributes() + + if telemetryDistroVersion, exists := attrs.Get(semconv.AttributeTelemetryDistroVersion); exists { + document.AddString("agent.version", telemetryDistroVersion.Str()) + return + } + + if telemetrySdkVersion, exists := attrs.Get(semconv.AttributeTelemetrySDKVersion); exists { + document.AddString("agent.version", telemetrySdkVersion.Str()) + return + } +} + +func encodeLogHostOsTypeECSMode(document *objmodel.Document, resource pcommon.Resource) { + // https://www.elastic.co/guide/en/ecs/current/ecs-os.html#field-os-type: + // + // "One of these following values should be used (lowercase): linux, macos, unix, windows. + // If the OS you’re dealing with is not in the list, the field should not be populated." + + var ecsHostOsType string + if semConvOsType, exists := resource.Attributes().Get(semconv.AttributeOSType); exists { + switch semConvOsType.Str() { + case "windows", "linux": + ecsHostOsType = semConvOsType.Str() + case "darwin": + ecsHostOsType = "macos" + case "aix", "hpux", "solaris": + ecsHostOsType = "unix" + } + } + + if semConvOsName, exists := resource.Attributes().Get(semconv.AttributeOSName); exists { + switch semConvOsName.Str() { + case "Android": + ecsHostOsType = "android" + case "iOS": + ecsHostOsType = "ios" + } + } + + if ecsHostOsType == "" { + return + } + document.AddString("host.os.type", ecsHostOsType) +} + +func encodeLogTimestampECSMode(document *objmodel.Document, record plog.LogRecord) { + if record.Timestamp() != 0 { + document.AddTimestamp("@timestamp", record.Timestamp()) + return + } + + document.AddTimestamp("@timestamp", record.ObservedTimestamp()) +} diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index fc6bff486f85..266c5dfecb59 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -13,7 +13,7 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/ptrace" - semconv "go.opentelemetry.io/collector/semconv/v1.18.0" + semconv "go.opentelemetry.io/collector/semconv/v1.22.0" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter/internal/objmodel" ) @@ -217,3 +217,533 @@ func TestEncodeEvents(t *testing.T) { }) } } + +func TestEncodeLogECSMode(t *testing.T) { + resource := pcommon.NewResource() + err := resource.Attributes().FromRaw(map[string]any{ + semconv.AttributeServiceName: "foo.bar", + semconv.AttributeServiceVersion: "1.1.0", + semconv.AttributeServiceInstanceID: "i-103de39e0a", + semconv.AttributeTelemetrySDKName: "opentelemetry", + semconv.AttributeTelemetrySDKVersion: "7.9.12", + semconv.AttributeTelemetrySDKLanguage: "perl", + semconv.AttributeCloudProvider: "gcp", + semconv.AttributeCloudAccountID: "19347013", + semconv.AttributeCloudRegion: "us-west-1", + semconv.AttributeCloudAvailabilityZone: "us-west-1b", + semconv.AttributeCloudPlatform: "gke", + semconv.AttributeContainerName: "happy-seger", + semconv.AttributeContainerID: "e69cc5d3dda", + semconv.AttributeContainerImageName: "my-app", + semconv.AttributeContainerRuntime: "docker", + semconv.AttributeHostName: "i-103de39e0a.gke.us-west-1b.cloud.google.com", + semconv.AttributeHostID: "i-103de39e0a", + semconv.AttributeHostType: "t2.medium", + semconv.AttributeHostArch: "x86_64", + semconv.AttributeProcessPID: 9833, + semconv.AttributeProcessCommandLine: "/usr/bin/ssh -l user 10.0.0.16", + semconv.AttributeProcessExecutablePath: "/usr/bin/ssh", + semconv.AttributeProcessRuntimeName: "OpenJDK Runtime Environment", + semconv.AttributeProcessRuntimeVersion: "14.0.2", + semconv.AttributeOSType: "darwin", + semconv.AttributeOSDescription: "Mac OS Mojave", + semconv.AttributeOSName: "Mac OS X", + semconv.AttributeOSVersion: "10.14.1", + semconv.AttributeDeviceID: "00000000-54b3-e7c7-0000-000046bffd97", + semconv.AttributeDeviceModelIdentifier: "SM-G920F", + semconv.AttributeDeviceModelName: "Samsung Galaxy S6", + semconv.AttributeDeviceManufacturer: "Samsung", + "k8s.namespace.name": "default", + "k8s.node.name": "node-1", + "k8s.pod.name": "opentelemetry-pod-autoconf", + "k8s.pod.uid": "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff", + }) + require.NoError(t, err) + + resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags) + err = resourceContainerImageTags.FromRaw([]any{"v3.4.0"}) + require.NoError(t, err) + + scope := pcommon.NewInstrumentationScope() + + record := plog.NewLogRecord() + err = record.Attributes().FromRaw(map[string]any{ + "event.name": "user-password-change", + }) + require.NoError(t, err) + observedTimestamp := pcommon.Timestamp(1710273641123456789) + record.SetObservedTimestamp(observedTimestamp) + + m := encodeModel{} + doc := m.encodeLogECSMode(resource, record, scope) + + expectedDocFields := pcommon.NewMap() + err = expectedDocFields.FromRaw(map[string]any{ + "service.name": "foo.bar", + "service.version": "1.1.0", + "service.node.name": "i-103de39e0a", + "agent.name": "opentelemetry/perl", + "agent.version": "7.9.12", + "cloud.provider": "gcp", + "cloud.account.id": "19347013", + "cloud.region": "us-west-1", + "cloud.availability_zone": "us-west-1b", + "cloud.service.name": "gke", + "container.name": "happy-seger", + "container.id": "e69cc5d3dda", + "container.image.name": "my-app", + "container.runtime": "docker", + "host.hostname": "i-103de39e0a.gke.us-west-1b.cloud.google.com", + "host.id": "i-103de39e0a", + "host.type": "t2.medium", + "host.architecture": "x86_64", + "process.pid": 9833, + "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", + "process.executable": "/usr/bin/ssh", + "service.runtime.name": "OpenJDK Runtime Environment", + "service.runtime.version": "14.0.2", + "host.os.platform": "darwin", + "host.os.full": "Mac OS Mojave", + "host.os.name": "Mac OS X", + "host.os.version": "10.14.1", + "host.os.type": "macos", + "device.id": "00000000-54b3-e7c7-0000-000046bffd97", + "device.model.identifier": "SM-G920F", + "device.model.name": "Samsung Galaxy S6", + "device.manufacturer": "Samsung", + "event.action": "user-password-change", + "kubernetes.namespace": "default", + "kubernetes.node.name": "node-1", + "kubernetes.pod.name": "opentelemetry-pod-autoconf", + "kubernetes.pod.uid": "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff", + }) + require.NoError(t, err) + + expectedDoc := objmodel.Document{} + expectedDoc.AddAttributes("", expectedDocFields) + expectedDoc.AddTimestamp("@timestamp", observedTimestamp) + expectedDoc.Add("container.image.tag", objmodel.ArrValue(objmodel.StringValue("v3.4.0"))) + + doc.Sort() + expectedDoc.Sort() + require.Equal(t, expectedDoc, doc) +} + +func TestEncodeLogECSModeAgentName(t *testing.T) { + tests := map[string]struct { + telemetrySdkName string + telemetrySdkLanguage string + telemetryDistroName string + + expectedAgentName string + expectedServiceLanguageName string + }{ + "none_set": { + expectedAgentName: "otlp", + expectedServiceLanguageName: "unknown", + }, + "name_set": { + telemetrySdkName: "opentelemetry", + expectedAgentName: "opentelemetry", + expectedServiceLanguageName: "unknown", + }, + "language_set": { + telemetrySdkLanguage: "java", + expectedAgentName: "otlp/java", + expectedServiceLanguageName: "java", + }, + "distro_set": { + telemetryDistroName: "parts-unlimited-java", + expectedAgentName: "otlp/unknown/parts-unlimited-java", + expectedServiceLanguageName: "unknown", + }, + "name_language_set": { + telemetrySdkName: "opentelemetry", + telemetrySdkLanguage: "java", + expectedAgentName: "opentelemetry/java", + expectedServiceLanguageName: "java", + }, + "name_distro_set": { + telemetrySdkName: "opentelemetry", + telemetryDistroName: "parts-unlimited-java", + expectedAgentName: "opentelemetry/unknown/parts-unlimited-java", + expectedServiceLanguageName: "unknown", + }, + "language_distro_set": { + telemetrySdkLanguage: "java", + telemetryDistroName: "parts-unlimited-java", + expectedAgentName: "otlp/java/parts-unlimited-java", + expectedServiceLanguageName: "java", + }, + "name_language_distro_set": { + telemetrySdkName: "opentelemetry", + telemetrySdkLanguage: "java", + telemetryDistroName: "parts-unlimited-java", + expectedAgentName: "opentelemetry/java/parts-unlimited-java", + expectedServiceLanguageName: "java", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + resource := pcommon.NewResource() + scope := pcommon.NewInstrumentationScope() + record := plog.NewLogRecord() + + if test.telemetrySdkName != "" { + resource.Attributes().PutStr(semconv.AttributeTelemetrySDKName, test.telemetrySdkName) + } + if test.telemetrySdkLanguage != "" { + resource.Attributes().PutStr(semconv.AttributeTelemetrySDKLanguage, test.telemetrySdkLanguage) + } + if test.telemetryDistroName != "" { + resource.Attributes().PutStr(semconv.AttributeTelemetryDistroName, test.telemetryDistroName) + } + + timestamp := pcommon.Timestamp(1710373859123456789) + record.SetTimestamp(timestamp) + + m := encodeModel{} + doc := m.encodeLogECSMode(resource, record, scope) + + expectedDoc := objmodel.Document{} + expectedDoc.AddTimestamp("@timestamp", timestamp) + expectedDoc.AddString("agent.name", test.expectedAgentName) + + doc.Sort() + expectedDoc.Sort() + require.Equal(t, expectedDoc, doc) + }) + } +} + +func TestEncodeLogECSModeAgentVersion(t *testing.T) { + tests := map[string]struct { + telemetryDistroVersion string + telemetrySdkVersion string + expectedAgentVersion string + }{ + "none_set": { + expectedAgentVersion: "", + }, + "distro_version_set": { + telemetryDistroVersion: "7.9.2", + expectedAgentVersion: "7.9.2", + }, + "sdk_version_set": { + telemetrySdkVersion: "8.10.3", + expectedAgentVersion: "8.10.3", + }, + "both_set": { + telemetryDistroVersion: "7.9.2", + telemetrySdkVersion: "8.10.3", + expectedAgentVersion: "7.9.2", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + resource := pcommon.NewResource() + scope := pcommon.NewInstrumentationScope() + record := plog.NewLogRecord() + + if test.telemetryDistroVersion != "" { + resource.Attributes().PutStr(semconv.AttributeTelemetryDistroVersion, test.telemetryDistroVersion) + } + if test.telemetrySdkVersion != "" { + resource.Attributes().PutStr(semconv.AttributeTelemetrySDKVersion, test.telemetrySdkVersion) + } + + timestamp := pcommon.Timestamp(1710373859123456789) + record.SetTimestamp(timestamp) + + m := encodeModel{} + doc := m.encodeLogECSMode(resource, record, scope) + + expectedDoc := objmodel.Document{} + expectedDoc.AddTimestamp("@timestamp", timestamp) + expectedDoc.AddString("agent.name", "otlp") + expectedDoc.AddString("agent.version", test.expectedAgentVersion) + + doc.Sort() + expectedDoc.Sort() + require.Equal(t, expectedDoc, doc) + }) + } +} + +func TestEncodeLogECSModeHostOSType(t *testing.T) { + tests := map[string]struct { + osType string + osName string + + expectedHostOsName string + expectedHostOsType string + expectedHostOsPlatform string + }{ + "none_set": { + expectedHostOsName: "", // should not be set + expectedHostOsType: "", // should not be set + expectedHostOsPlatform: "", // should not be set + }, + "type_windows": { + osType: "windows", + expectedHostOsName: "", // should not be set + expectedHostOsType: "windows", + expectedHostOsPlatform: "windows", + }, + "type_linux": { + osType: "linux", + expectedHostOsName: "", // should not be set + expectedHostOsType: "linux", + expectedHostOsPlatform: "linux", + }, + "type_darwin": { + osType: "darwin", + expectedHostOsName: "", // should not be set + expectedHostOsType: "macos", + expectedHostOsPlatform: "darwin", + }, + "type_aix": { + osType: "aix", + expectedHostOsName: "", // should not be set + expectedHostOsType: "unix", + expectedHostOsPlatform: "aix", + }, + "type_hpux": { + osType: "hpux", + expectedHostOsName: "", // should not be set + expectedHostOsType: "unix", + expectedHostOsPlatform: "hpux", + }, + "type_solaris": { + osType: "solaris", + expectedHostOsName: "", // should not be set + expectedHostOsType: "unix", + expectedHostOsPlatform: "solaris", + }, + "type_unknown": { + osType: "unknown", + expectedHostOsName: "", // should not be set + expectedHostOsType: "", // should not be set + expectedHostOsPlatform: "unknown", + }, + "name_android": { + osName: "Android", + expectedHostOsName: "Android", + expectedHostOsType: "android", + expectedHostOsPlatform: "", // should not be set + }, + "name_ios": { + osName: "iOS", + expectedHostOsName: "iOS", + expectedHostOsType: "ios", + expectedHostOsPlatform: "", // should not be set + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + resource := pcommon.NewResource() + scope := pcommon.NewInstrumentationScope() + record := plog.NewLogRecord() + + if test.osType != "" { + resource.Attributes().PutStr(semconv.AttributeOSType, test.osType) + } + if test.osName != "" { + resource.Attributes().PutStr(semconv.AttributeOSName, test.osName) + } + + timestamp := pcommon.Timestamp(1710373859123456789) + record.SetTimestamp(timestamp) + + m := encodeModel{} + doc := m.encodeLogECSMode(resource, record, scope) + + expectedDoc := objmodel.Document{} + expectedDoc.AddTimestamp("@timestamp", timestamp) + expectedDoc.AddString("agent.name", "otlp") + if test.expectedHostOsName != "" { + expectedDoc.AddString("host.os.name", test.expectedHostOsName) + } + if test.expectedHostOsType != "" { + expectedDoc.AddString("host.os.type", test.expectedHostOsType) + } + if test.expectedHostOsPlatform != "" { + expectedDoc.AddString("host.os.platform", test.expectedHostOsPlatform) + } + + doc.Sort() + expectedDoc.Sort() + require.Equal(t, expectedDoc, doc) + }) + } +} + +func TestEncodeLogECSModeTimestamps(t *testing.T) { + tests := map[string]struct { + timeUnixNano int64 + observedTimeUnixNano int64 + expectedTimestamp time.Time + }{ + "only_observed_set": { + observedTimeUnixNano: 1710273641123456789, + expectedTimestamp: time.Unix(0, 1710273641123456789), + }, + "both_set": { + timeUnixNano: 1710273639345678901, + observedTimeUnixNano: 1710273641123456789, + expectedTimestamp: time.Unix(0, 1710273639345678901), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + resource := pcommon.NewResource() + scope := pcommon.NewInstrumentationScope() + record := plog.NewLogRecord() + + if test.timeUnixNano > 0 { + record.SetTimestamp(pcommon.Timestamp(test.timeUnixNano)) + } + if test.observedTimeUnixNano > 0 { + record.SetObservedTimestamp(pcommon.Timestamp(test.observedTimeUnixNano)) + } + + m := encodeModel{} + doc := m.encodeLogECSMode(resource, record, scope) + + expectedDoc := objmodel.Document{} + expectedDoc.AddTimestamp("@timestamp", pcommon.NewTimestampFromTime(test.expectedTimestamp)) + expectedDoc.AddString("agent.name", "otlp") + + doc.Sort() + expectedDoc.Sort() + require.Equal(t, expectedDoc, doc) + }) + } +} + +func TestMapLogAttributesToECS(t *testing.T) { + tests := map[string]struct { + attrs func() pcommon.Map + conversionMap map[string]string + expectedDoc func() objmodel.Document + }{ + "no_attrs": { + attrs: pcommon.NewMap, + conversionMap: map[string]string{ + "foo.bar": "baz", + }, + expectedDoc: func() objmodel.Document { + return objmodel.Document{} + }, + }, + "no_conversion_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + return m + }, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("foo.bar", "baz") + return d + }, + }, + "empty_conversion_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + return m + }, + conversionMap: map[string]string{}, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("foo.bar", "baz") + return d + }, + }, + "all_attrs_in_conversion_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + m.PutInt("qux", 17) + return m + }, + conversionMap: map[string]string{ + "foo.bar": "bar.qux", + "qux": "foo", + }, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("bar.qux", "baz") + d.AddInt("foo", 17) + return d + }, + }, + "some_attrs_in_conversion_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + m.PutInt("qux", 17) + return m + }, + conversionMap: map[string]string{ + "foo.bar": "bar.qux", + }, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("bar.qux", "baz") + d.AddInt("qux", 17) + return d + }, + }, + "no_attrs_in_conversion_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + m.PutInt("qux", 17) + return m + }, + conversionMap: map[string]string{ + "baz": "qux", + }, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("foo.bar", "baz") + d.AddInt("qux", 17) + return d + }, + }, + "extra_keys_in_conversion_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + return m + }, + conversionMap: map[string]string{ + "foo.bar": "bar.qux", + "qux": "foo", + }, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("bar.qux", "baz") + return d + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var doc objmodel.Document + encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap) + + doc.Sort() + expectedDoc := test.expectedDoc() + expectedDoc.Sort() + require.Equal(t, expectedDoc, doc) + }) + } +}