feat: add ability to filter and export markers (open-telemetry#27862)

**Description:** This add logic to filter logs based on log conditions and sent desired logs as event markers to the honeycomb marker api. **Link to tracking Issue:** open-telemetry#27666 **Testing:** Unit testing for log exporter and config. Added component testing to `otelcontribcol`. **Documentation:** README describing component usage Screenshot of exported markers showing up in Honeycomb <img width="1225" alt="Screenshot 2023-11-14 at 1 27 49 PM" src="https://github.com/open-telemetry/opentelemetry-collector-contrib/assets/35741033/128d689a-cf1e-4959-9df3-6c88248a7fdb"> --------- Co-authored-by: Tyler Helmuth <[email protected]>
graphaelli · Nov 17, 2023 · 7118395 · 7118395
1 parent d6de55d
commit 7118395
Show file tree

Hide file tree

Showing 13 changed files with 551 additions and 129 deletions.
diff --git a/cmd/otelcontribcol/builder-config.yaml b/cmd/otelcontribcol/builder-config.yaml
@@ -56,6 +56,7 @@ exporters:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudexporter v0.89.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudpubsubexporter v0.89.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlemanagedprometheusexporter v0.89.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/honeycombmarkerexporter v0.89.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/influxdbexporter v0.89.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/instanaexporter v0.89.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.89.0
@@ -389,6 +390,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/googlecloudpubsubreceiver => ../../receiver/googlecloudpubsubreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sumologicexporter => ../../exporter/sumologicexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/instanaexporter => ../../exporter/instanaexporter
+  - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/honeycombmarkerexporter => ../../exporter/honeycombmarkerexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otlpjsonfilereceiver => ../../receiver/otlpjsonfilereceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor => ../../processor/redactionprocessor
   - github.com/open-telemetry/opentelemetry-collector-contrib/extension/jaegerremotesampling => ../../extension/jaegerremotesampling

diff --git a/cmd/otelcontribcol/components.go b/cmd/otelcontribcol/components.go
diff --git a/cmd/otelcontribcol/exporters_test.go b/cmd/otelcontribcol/exporters_test.go
@@ -40,6 +40,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/f5cloudexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/honeycombmarkerexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/influxdbexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/instanaexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter"
@@ -421,6 +422,18 @@ func TestDefaultExporters(t *testing.T) {
 			exporter:      "googlecloudpubsub",
 			skipLifecycle: true,
 		},
+		{
+			exporter: "honeycombmarker",
+			getConfigFn: func() component.Config {
+				cfg := expFactories["honeycombmarker"].CreateDefaultConfig().(*honeycombmarkerexporter.Config)
+				cfg.Endpoint = "http://" + endpoint
+				// disable queue to validate passing the test data synchronously
+				cfg.QueueSettings.Enabled = false
+				cfg.RetrySettings.Enabled = false
+				return cfg
+			},
+			expectConsumeErr: true,
+		},
 		{
 			exporter: "influxdb",
 			getConfigFn: func() component.Config {

diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod
@@ -32,6 +32,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudexporter v0.89.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudpubsubexporter v0.89.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlemanagedprometheusexporter v0.89.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/honeycombmarkerexporter v0.89.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/influxdbexporter v0.89.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/instanaexporter v0.89.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.89.0
@@ -1082,6 +1083,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sumol
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/instanaexporter => ../../exporter/instanaexporter
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/honeycombmarkerexporter => ../../exporter/honeycombmarkerexporter
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otlpjsonfilereceiver => ../../receiver/otlpjsonfilereceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor => ../../processor/redactionprocessor

diff --git a/exporter/honeycombmarkerexporter/README.md b/exporter/honeycombmarkerexporter/README.md
@@ -5,38 +5,27 @@ This exporter allows creating markers, via the Honeycomb Markers API, based on t
 
 The following configuration options are supported:
 
-* `api_key` (Required): This is the API key (also called Write Key) for your Honeycomb account.
-* `api_url` (Required): You can set the hostname to send marker data to.
-* `markers` (Required): This specifies the exact configuration to create an event marker. 
-  * `type`: Specifies the marker type. Markers with the same type will appear in the same color in Honeycomb. MarkerType or MarkerColor should be set.  
-  * `color`: Specifies the marker color. Will only be used if MarkerType is not set.
-  * `messagefield`: This is the attribute that will be used as the message. If necessary the value will be converted to a string.
-  * `urlfield`: This is the attribute that will be used as the url. If necessary the value will be converted to a string.
-  * `rules`: This is a list of OTTL rules that determine when to create an event marker. 
-    * `resourceconditions`: A list of ottlresource conditions that determine a match
-    * `logconditions`: A list of ottllog conditions that determine a match
+* `api_key` (Required): This is the API key for your Honeycomb account.
+* `api_url` (Required): This sets the hostname to send marker data to.
+* `markers` (Required): This is a list of configurations to create an event marker. 
+  * `type` (Required): Specifies the marker type. 
+  * `message_key`: This attribute will be used as the message. It describes the event marker. If necessary the value will be converted to a string.
+  * `url_key`: This attribute will be used as the url. It can be accessed through the event marker in Honeycomb. If necessary the value will be converted to a string.
+  * `rules` (Required): This is a list of OTTL rules that determine when to create an event marker. 
+    * `log_conditions` (Required): A list of ottllog conditions that determine a match
   Example:
 
 ```yaml
 exporters:
-  honeycomb:
-    api_key: "my-api-key"
-    api_url: "https://api.testhost.io"
+  honeycombmarker:
+    api_key: "environment-api-key"
+    api_url: "https://api.honeycomb.io"
     markers:
-      - type: "fooType",
-        messagefield: "test message",
-        urlfield: "https://api.testhost.io",
+      - type: "marker-type"
+        message_key: "marker-message"
+        url_key: "marker-url"
+        dataset_slug: "__all__"
         rules:
-          - resourceconditions:
-              - IsMatch(attributes["test"], ".*")
-          - logconditions:
-              - body == "test"
-      - color: "green",
-        messagefield: "another test message",
-        urlfield: "https://api.testhost.io",
-        rules:
-          - resourceconditions:
-              - IsMatch(attributes["test"], ".*")
-          - logconditions:
+          - log_conditions:
               - body == "test"
 ```
diff --git a/exporter/honeycombmarkerexporter/config.go b/exporter/honeycombmarkerexporter/config.go
@@ -7,11 +7,15 @@ import (
 	"fmt"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/configopaque"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 	"go.uber.org/zap"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter/expr"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter/filterottl"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottllog"
 )
 
 // Config defines configuration for the Honeycomb Marker exporter.
@@ -24,58 +28,60 @@ type Config struct {
 
 	// Markers is the list of markers to create
 	Markers []Marker `mapstructure:"markers"`
+
+	confighttp.HTTPClientSettings `mapstructure:",squash"`
+	exporterhelper.QueueSettings  `mapstructure:"sending_queue"`
+	exporterhelper.RetrySettings  `mapstructure:"retry_on_failure"`
 }
 
 type Marker struct {
-	// Type defines the type of Marker.  Markers with the same type appear in Honeycomb with the same color
+	// Type defines the type of Marker.
 	Type string `mapstructure:"type"`
 
-	// Color is the color of the Marker. Will only be used if the Type does not already exist.
-	Color string `mapstructure:"color"`
-
-	// MessageField is the attribute that will be used as the message.
+	// MessageKey is the attribute that will be used as the message.
 	// If necessary the value will be converted to a string.
-	MessageField string `mapstructure:"message_field"`
+	MessageKey string `mapstructure:"message_key"`
 
-	// URLField is the attribute that will be used as the url.
+	// URLKey is the attribute that will be used as the url.
 	// If necessary the value will be converted to a string.
-	URLField string `mapstructure:"url_field"`
+	URLKey string `mapstructure:"url_key"`
 
 	// Rules are the OTTL rules that determine when a piece of telemetry should be turned into a Marker
 	Rules Rules `mapstructure:"rules"`
+
+	// DatasetSlug is the endpoint that specifies the Honeycomb environment
+	DatasetSlug string `mapstructure:"dataset_slug"`
 }
 
 type Rules struct {
-	// ResourceConditions is the list of ottlresource conditions that determine a match
-	ResourceConditions []string `mapstructure:"resource_conditions"`
-
 	// LogConditions is the list of ottllog conditions that determine a match
 	LogConditions []string `mapstructure:"log_conditions"`
+
+	logBoolExpr expr.BoolExpr[ottllog.TransformContext]
 }
 
-var defaultCfg = createDefaultConfig().(*Config)
+var _ component.Config = (*Config)(nil)
 
 func (cfg *Config) Validate() error {
-	if cfg == nil {
-		cfg = defaultCfg
-	}
-
 	if cfg.APIKey == "" {
 		return fmt.Errorf("invalid API Key")
 	}
 
 	if len(cfg.Markers) != 0 {
 		for _, m := range cfg.Markers {
-			if len(m.Rules.ResourceConditions) == 0 && len(m.Rules.LogConditions) == 0 {
-				return fmt.Errorf("no rules supplied for Marker %v", m)
+			if m.Type == "" {
+				return fmt.Errorf("marker must have a type %v", m)
 			}
 
-			_, err := filterottl.NewBoolExprForResource(m.Rules.ResourceConditions, filterottl.StandardResourceFuncs(), ottl.PropagateError, component.TelemetrySettings{Logger: zap.NewNop()})
-			if err != nil {
-				return err
+			if m.DatasetSlug == "" {
+				return fmt.Errorf("marker must have a dataset slug %v", m)
+			}
+
+			if len(m.Rules.LogConditions) == 0 {
+				return fmt.Errorf("marker must have rules %v", m)
 			}
 
-			_, err = filterottl.NewBoolExprForLog(m.Rules.LogConditions, filterottl.StandardLogFuncs(), ottl.PropagateError, component.TelemetrySettings{Logger: zap.NewNop()})
+			_, err := filterottl.NewBoolExprForLog(m.Rules.LogConditions, filterottl.StandardLogFuncs(), ottl.PropagateError, component.TelemetrySettings{Logger: zap.NewNop()})
 			if err != nil {
 				return err
 			}

diff --git a/exporter/honeycombmarkerexporter/config_test.go b/exporter/honeycombmarkerexporter/config_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/honeycombmarkerexporter/internal/metadata"
 )
@@ -26,45 +27,23 @@ func TestLoadConfig(t *testing.T) {
 		expected component.Config
 	}{
 		{
-			id: component.NewIDWithName("honeycomb", ""),
+			id: component.NewIDWithName(metadata.Type, ""),
 			expected: &Config{
-				APIKey: "test-apikey",
-				APIURL: "https://api.testhost.io",
+				QueueSettings: exporterhelper.NewDefaultQueueSettings(),
+				RetrySettings: exporterhelper.NewDefaultRetrySettings(),
+				APIKey:        "test-apikey",
+				APIURL:        "https://api.testhost.io",
 				Markers: []Marker{
 					{
-						Type:         "fooType",
-						MessageField: "test message",
-						URLField:     "https://api.testhost.io",
+						Type:       "fooType",
+						MessageKey: "test message",
+						URLKey:     "https://api.testhost.io",
 						Rules: Rules{
-							ResourceConditions: []string{
-								`IsMatch(attributes["test"], ".*")`,
-							},
-							LogConditions: []string{
-								`body == "test"`,
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			id: component.NewIDWithName("honeycomb", "color_no_type"),
-			expected: &Config{
-				APIKey: "test-apikey",
-				APIURL: "https://api.testhost.io",
-				Markers: []Marker{
-					{
-						Color:        "green",
-						MessageField: "test message",
-						URLField:     "https://api.testhost.io",
-						Rules: Rules{
-							ResourceConditions: []string{
-								`IsMatch(attributes["test"], ".*")`,
-							},
 							LogConditions: []string{
 								`body == "test"`,
 							},
 						},
+						DatasetSlug: "__all__",
 					},
 				},
 			},
@@ -81,6 +60,9 @@ func TestLoadConfig(t *testing.T) {
 		{
 			id: component.NewIDWithName(metadata.Type, "no_markers_supplied"),
 		},
+		{
+			id: component.NewIDWithName(metadata.Type, "no_dataset_slug"),
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/exporter/honeycombmarkerexporter/factory.go b/exporter/honeycombmarkerexporter/factory.go
@@ -36,12 +36,19 @@ func createLogsExporter(
 ) (exporter.Logs, error) {
 	cf := cfg.(*Config)
 
-	exporter := newLogsExporter(set.Logger, cf)
+	logsExp, err := newHoneycombLogsExporter(set.TelemetrySettings, cf)
+	if err != nil {
+		return nil, err
+	}
 
 	return exporterhelper.NewLogsExporter(
 		ctx,
 		set,
 		cfg,
-		exporter.exportLogs,
+		logsExp.exportMarkers,
+		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
+		exporterhelper.WithRetry(cf.RetrySettings),
+		exporterhelper.WithQueue(cf.QueueSettings),
+		exporterhelper.WithStart(logsExp.start),
 	)
 }
diff --git a/exporter/honeycombmarkerexporter/go.mod b/exporter/honeycombmarkerexporter/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.89.0
 	github.com/stretchr/testify v1.8.4
 	go.opentelemetry.io/collector/component v0.89.0
+	go.opentelemetry.io/collector/config/confighttp v0.89.0
 	go.opentelemetry.io/collector/config/configopaque v0.89.0
 	go.opentelemetry.io/collector/confmap v0.89.0
 	go.opentelemetry.io/collector/exporter v0.89.0
@@ -18,14 +19,20 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.0.1 // indirect
@@ -36,13 +43,20 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.89.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rs/cors v1.10.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/collector v0.89.0 // indirect
+	go.opentelemetry.io/collector/config/configauth v0.89.0 // indirect
+	go.opentelemetry.io/collector/config/configcompression v0.89.0 // indirect
 	go.opentelemetry.io/collector/config/configtelemetry v0.89.0 // indirect
+	go.opentelemetry.io/collector/config/configtls v0.89.0 // indirect
+	go.opentelemetry.io/collector/config/internal v0.89.0 // indirect
 	go.opentelemetry.io/collector/consumer v0.89.0 // indirect
 	go.opentelemetry.io/collector/extension v0.89.0 // indirect
+	go.opentelemetry.io/collector/extension/auth v0.89.0 // indirect
 	go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect
 	go.opentelemetry.io/collector/receiver v0.89.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.20.0 // indirect
 	go.opentelemetry.io/otel/metric v1.20.0 // indirect
 	go.opentelemetry.io/otel/trace v1.20.0 // indirect