[Feature][exporter/azuredataexplorerexporter] Exporter helper support (…

…open-telemetry#27272) **Description:** <Describe what has changed.>  Added support for Exporter Helper configuration. **Link to tracking Issue:** open-telemetry#24329 **Testing:** <Describe what testing was performed and which tests were added.> Added tests and manually tested with e2e scenarios --------- Co-authored-by: Ramachandran A G <[email protected]> Co-authored-by: Ziqi Zhao <[email protected]> Co-authored-by: Ramachandran A G <[email protected]>
openinsight-proj · Oct 18, 2023 · 99f3758 · 99f3758
1 parent 049dadf
commit 99f3758
Show file tree

Hide file tree

Showing 7 changed files with 149 additions and 18 deletions.
diff --git a/.chloggen/ingestionretry.yaml b/.chloggen/ingestionretry.yaml
@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: azuredataexplorerexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Added exporter helper config support for Azure Data Explorer exporter
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [24329]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]
diff --git a/exporter/azuredataexplorerexporter/README.md b/exporter/azuredataexplorerexporter/README.md
@@ -16,6 +16,7 @@ This exporter sends metrics, logs and trace data to
   [Azure Data Explorer](https://docs.microsoft.com/en-us/azure/data-explorer),
   [Azure Synapse Data Explorer](https://docs.microsoft.com/en-us/azure/synapse-analytics/data-explorer/data-explorer-overview) and
   [Real time analytics in Fabric](https://learn.microsoft.com/en-us/fabric/real-time-analytics/overview)
+
 ## Configuration
 
 The following settings are required:
@@ -78,6 +79,17 @@ exporters:
     traces_table_json_mapping: "oteltraces_mapping"
     # Type of ingestion managed or queued
     ingestion_type : "managed"
+    #other available exporter helper options, see more here: https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md
+    # timeout: 10s
+    # sending_queue:
+    #   enabled: true
+    #   num_consumers: 2
+    #   queue_size: 10
+    # retry_on_failure:
+    #   enabled: true
+    #   initial_interval: 10s
+    #   max_interval: 60s
+    #   max_elapsed_time: 10m
 ```
 
 ## Attribute mapping
@@ -193,3 +205,24 @@ with ( docstring = "Histo sum count processing function", folder = "UpdatePolicy
 .alter table HistoData policy update 
 @'[{ "IsEnabled": true, "Source": "RawMetricsData","Query": "ExtractHistoCountColumns()", "IsTransactional": false, "PropagateIngestionProperties": false}]'
 ```
+
+### Opentelemetry Exporter Helper Configurations
+
+The ADX exporter now includes support for Opentelemetry exporter helper configurations. This feature allows you to leverage the exporter helper capabilities(retries, timeout etc.) provided natively by Otel. Read more [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md).
+
+Please note that this configuration is not enabled by default. To utilize the Opentelemetry exporter helper, you will need to add it manually to the configuration.
+
+#### Example Configuration
+
+```yaml
+# Example Opentelemetry Exporter Configuration
+  timeout: 10s
+  sending_queue:
+      enabled: true
+      num_consumers: 2
+      queue_size: 10
+  retry_on_failure:
+      enabled: true
+      initial_interval: 10s
+      max_interval: 60s
+      max_elapsed_time: 10m
diff --git a/exporter/azuredataexplorerexporter/adx_exporter_test.go b/exporter/azuredataexplorerexporter/adx_exporter_test.go
@@ -207,7 +207,8 @@ func TestCreateKcsb(t *testing.T) {
 			applicationID:     "",
 		},
 	}
-	for _, tt := range tests {
+	for i := range tests {
+		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			wantAppID := tt.applicationID
 			gotKcsb := createKcsb(&tt.config, "1.0.0")

diff --git a/exporter/azuredataexplorerexporter/config.go b/exporter/azuredataexplorerexporter/config.go
@@ -10,23 +10,27 @@ import (
 
 	"github.com/google/uuid"
 	"go.opentelemetry.io/collector/config/configopaque"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 )
 
 // Config defines configuration for Azure Data Explorer Exporter
 type Config struct {
-	ClusterURI         string              `mapstructure:"cluster_uri"`
-	ApplicationID      string              `mapstructure:"application_id"`
-	ApplicationKey     configopaque.String `mapstructure:"application_key"`
-	TenantID           string              `mapstructure:"tenant_id"`
-	ManagedIdentityID  string              `mapstructure:"managed_identity_id"`
-	Database           string              `mapstructure:"db_name"`
-	MetricTable        string              `mapstructure:"metrics_table_name"`
-	LogTable           string              `mapstructure:"logs_table_name"`
-	TraceTable         string              `mapstructure:"traces_table_name"`
-	MetricTableMapping string              `mapstructure:"metrics_table_json_mapping"`
-	LogTableMapping    string              `mapstructure:"logs_table_json_mapping"`
-	TraceTableMapping  string              `mapstructure:"traces_table_json_mapping"`
-	IngestionType      string              `mapstructure:"ingestion_type"`
+	exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	exporterhelper.QueueSettings   `mapstructure:"sending_queue"`
+	exporterhelper.RetrySettings   `mapstructure:"retry_on_failure"`
+	ClusterURI                     string              `mapstructure:"cluster_uri"`
+	ApplicationID                  string              `mapstructure:"application_id"`
+	ApplicationKey                 configopaque.String `mapstructure:"application_key"`
+	TenantID                       string              `mapstructure:"tenant_id"`
+	ManagedIdentityID              string              `mapstructure:"managed_identity_id"`
+	Database                       string              `mapstructure:"db_name"`
+	MetricTable                    string              `mapstructure:"metrics_table_name"`
+	LogTable                       string              `mapstructure:"logs_table_name"`
+	TraceTable                     string              `mapstructure:"traces_table_name"`
+	MetricTableMapping             string              `mapstructure:"metrics_table_json_mapping"`
+	LogTableMapping                string              `mapstructure:"logs_table_json_mapping"`
+	TraceTableMapping              string              `mapstructure:"traces_table_json_mapping"`
+	IngestionType                  string              `mapstructure:"ingestion_type"`
 }
 
 // Validate checks if the exporter configuration is valid

diff --git a/exporter/azuredataexplorerexporter/config_test.go b/exporter/azuredataexplorerexporter/config_test.go
@@ -6,11 +6,13 @@ package azuredataexplorerexporter // import "github.com/open-telemetry/opentelem
 import (
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azuredataexplorerexporter/internal/metadata"
 )
@@ -80,6 +82,34 @@ func TestLoadConfig(t *testing.T) {
 			id:           component.NewIDWithName(metadata.Type, "7"),
 			errorMessage: `clusterURI config is mandatory`,
 		},
+		{
+			id: component.NewIDWithName(metadata.Type, "8"),
+			expected: &Config{
+				ClusterURI:     "https://CLUSTER.kusto.windows.net",
+				ApplicationID:  "f80da32c-108c-415c-a19e-643f461a677a",
+				ApplicationKey: "xx-xx-xx-xx",
+				TenantID:       "21ff9e36-fbaa-43c8-98ba-00431ea10bc3",
+				Database:       "oteldb",
+				MetricTable:    "OTELMetrics",
+				LogTable:       "OTELLogs",
+				TraceTable:     "OTELTraces",
+				IngestionType:  managedIngestType,
+				TimeoutSettings: exporterhelper.TimeoutSettings{
+					Timeout: 10 * time.Second,
+				},
+				RetrySettings: exporterhelper.RetrySettings{
+					Enabled:         true,
+					InitialInterval: 10 * time.Second,
+					MaxInterval:     60 * time.Second,
+					MaxElapsedTime:  10 * time.Minute,
+				},
+				QueueSettings: exporterhelper.QueueSettings{
+					Enabled:      true,
+					NumConsumers: 2,
+					QueueSize:    10,
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/exporter/azuredataexplorerexporter/factory.go b/exporter/azuredataexplorerexporter/factory.go
@@ -74,7 +74,9 @@ func createMetricsExporter(
 		set,
 		adxCfg,
 		adp.metricsDataPusher,
-		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
+		exporterhelper.WithTimeout(adxCfg.TimeoutSettings),
+		exporterhelper.WithRetry(adxCfg.RetrySettings),
+		exporterhelper.WithQueue(adxCfg.QueueSettings),
 		exporterhelper.WithShutdown(adp.Close))
 
 	if err != nil {
@@ -104,7 +106,9 @@ func createTracesExporter(
 		set,
 		adxCfg,
 		adp.tracesDataPusher,
-		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
+		exporterhelper.WithTimeout(adxCfg.TimeoutSettings),
+		exporterhelper.WithRetry(adxCfg.RetrySettings),
+		exporterhelper.WithQueue(adxCfg.QueueSettings),
 		exporterhelper.WithShutdown(adp.Close))
 
 	if err != nil {
@@ -134,7 +138,9 @@ func createLogsExporter(
 		set,
 		adxCfg,
 		adp.logsDataPusher,
-		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
+		exporterhelper.WithTimeout(adxCfg.TimeoutSettings),
+		exporterhelper.WithRetry(adxCfg.RetrySettings),
+		exporterhelper.WithQueue(adxCfg.QueueSettings),
 		exporterhelper.WithShutdown(adp.Close))
 
 	if err != nil {

diff --git a/exporter/azuredataexplorerexporter/testdata/config.yaml b/exporter/azuredataexplorerexporter/testdata/config.yaml
@@ -115,4 +115,34 @@ azuredataexplorer/7:
   # raw traces table
   traces_table_name: "OTELTraces"
   # type of ingestion managed or queued
-  ingestion_type: "managed"   
+  ingestion_type: "managed"  
+azuredataexplorer/8:
+  # Kusto cluster uri
+  cluster_uri: "https://CLUSTER.kusto.windows.net"
+  # Client Id
+  application_id: "f80da32c-108c-415c-a19e-643f461a677a"
+  # The client secret for the client
+  application_key: "xx-xx-xx-xx"
+  # The tenant
+  tenant_id: "21ff9e36-fbaa-43c8-98ba-00431ea10bc3"
+  # database for the logs
+  db_name: "oteldb"
+  # raw metric table name
+  metrics_table_name: "OTELMetrics"
+  # raw log table name
+  logs_table_name: "OTELLogs"
+  # raw traces table
+  traces_table_name: "OTELTraces"
+  # type of ingestion managed or queued
+  ingestion_type: "managed"
+  #export helper options
+  timeout: 10s
+  sending_queue:
+    enabled: true
+    num_consumers: 2
+    queue_size: 10
+  retry_on_failure:
+    enabled: true
+    initial_interval: 10s
+    max_interval: 60s
+    max_elapsed_time: 10m