From dd8e6f51e5780ff71c1d6a85251e948e932ba206 Mon Sep 17 00:00:00 2001 From: Abhishek Saharn <102726227+asaharn@users.noreply.github.com> Date: Thu, 12 Oct 2023 20:21:12 +0530 Subject: [PATCH] [Feature][exporter/azuredataexplorerexporter] Exporter helper support (#27272) **Description:** Added support for Exporter Helper configuration. **Link to tracking Issue:** #24329 **Testing:** Added tests and manually tested with e2e scenarios --------- Co-authored-by: Ramachandran A G Co-authored-by: Ziqi Zhao Co-authored-by: Ramachandran A G <106139410+ag-ramachandran@users.noreply.github.com> --- .chloggen/ingestionretry.yaml | 27 +++++++++++++++ exporter/azuredataexplorerexporter/README.md | 33 +++++++++++++++++++ .../adx_exporter_test.go | 3 +- exporter/azuredataexplorerexporter/config.go | 30 +++++++++-------- .../azuredataexplorerexporter/config_test.go | 30 +++++++++++++++++ exporter/azuredataexplorerexporter/factory.go | 12 +++++-- .../testdata/config.yaml | 32 +++++++++++++++++- 7 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 .chloggen/ingestionretry.yaml diff --git a/.chloggen/ingestionretry.yaml b/.chloggen/ingestionretry.yaml new file mode 100644 index 000000000000..499fdef2209f --- /dev/null +++ b/.chloggen/ingestionretry.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: azuredataexplorerexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Added exporter helper config support for Azure Data Explorer exporter + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [24329] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/azuredataexplorerexporter/README.md b/exporter/azuredataexplorerexporter/README.md index 6831e803a85d..b0024e90d34a 100644 --- a/exporter/azuredataexplorerexporter/README.md +++ b/exporter/azuredataexplorerexporter/README.md @@ -16,6 +16,7 @@ This exporter sends metrics, logs and trace data to [Azure Data Explorer](https://docs.microsoft.com/en-us/azure/data-explorer), [Azure Synapse Data Explorer](https://docs.microsoft.com/en-us/azure/synapse-analytics/data-explorer/data-explorer-overview) and [Real time analytics in Fabric](https://learn.microsoft.com/en-us/fabric/real-time-analytics/overview) + ## Configuration The following settings are required: @@ -78,6 +79,17 @@ exporters: traces_table_json_mapping: "oteltraces_mapping" # Type of ingestion managed or queued ingestion_type : "managed" + #other available exporter helper options, see more here: https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md + # timeout: 10s + # sending_queue: + # enabled: true + # num_consumers: 2 + # queue_size: 10 + # retry_on_failure: + # enabled: true + # initial_interval: 10s + # max_interval: 60s + # max_elapsed_time: 10m ``` ## Attribute mapping @@ -193,3 +205,24 @@ with ( docstring = "Histo sum count processing function", folder = "UpdatePolicy .alter table HistoData policy update @'[{ "IsEnabled": true, "Source": "RawMetricsData","Query": "ExtractHistoCountColumns()", "IsTransactional": false, "PropagateIngestionProperties": false}]' ``` + +### Opentelemetry Exporter Helper Configurations + +The ADX exporter now includes support for Opentelemetry exporter helper configurations. This feature allows you to leverage the exporter helper capabilities(retries, timeout etc.) provided natively by Otel. Read more [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md). + +Please note that this configuration is not enabled by default. To utilize the Opentelemetry exporter helper, you will need to add it manually to the configuration. + +#### Example Configuration + +```yaml +# Example Opentelemetry Exporter Configuration + timeout: 10s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 10 + retry_on_failure: + enabled: true + initial_interval: 10s + max_interval: 60s + max_elapsed_time: 10m diff --git a/exporter/azuredataexplorerexporter/adx_exporter_test.go b/exporter/azuredataexplorerexporter/adx_exporter_test.go index 06905aaf6c27..653c3626a8f8 100644 --- a/exporter/azuredataexplorerexporter/adx_exporter_test.go +++ b/exporter/azuredataexplorerexporter/adx_exporter_test.go @@ -207,7 +207,8 @@ func TestCreateKcsb(t *testing.T) { applicationID: "", }, } - for _, tt := range tests { + for i := range tests { + tt := tests[i] t.Run(tt.name, func(t *testing.T) { wantAppID := tt.applicationID gotKcsb := createKcsb(&tt.config, "1.0.0") diff --git a/exporter/azuredataexplorerexporter/config.go b/exporter/azuredataexplorerexporter/config.go index 4ffea154b453..084566b8c76e 100644 --- a/exporter/azuredataexplorerexporter/config.go +++ b/exporter/azuredataexplorerexporter/config.go @@ -10,23 +10,27 @@ import ( "github.com/google/uuid" "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/exporter/exporterhelper" ) // Config defines configuration for Azure Data Explorer Exporter type Config struct { - ClusterURI string `mapstructure:"cluster_uri"` - ApplicationID string `mapstructure:"application_id"` - ApplicationKey configopaque.String `mapstructure:"application_key"` - TenantID string `mapstructure:"tenant_id"` - ManagedIdentityID string `mapstructure:"managed_identity_id"` - Database string `mapstructure:"db_name"` - MetricTable string `mapstructure:"metrics_table_name"` - LogTable string `mapstructure:"logs_table_name"` - TraceTable string `mapstructure:"traces_table_name"` - MetricTableMapping string `mapstructure:"metrics_table_json_mapping"` - LogTableMapping string `mapstructure:"logs_table_json_mapping"` - TraceTableMapping string `mapstructure:"traces_table_json_mapping"` - IngestionType string `mapstructure:"ingestion_type"` + exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. + exporterhelper.QueueSettings `mapstructure:"sending_queue"` + exporterhelper.RetrySettings `mapstructure:"retry_on_failure"` + ClusterURI string `mapstructure:"cluster_uri"` + ApplicationID string `mapstructure:"application_id"` + ApplicationKey configopaque.String `mapstructure:"application_key"` + TenantID string `mapstructure:"tenant_id"` + ManagedIdentityID string `mapstructure:"managed_identity_id"` + Database string `mapstructure:"db_name"` + MetricTable string `mapstructure:"metrics_table_name"` + LogTable string `mapstructure:"logs_table_name"` + TraceTable string `mapstructure:"traces_table_name"` + MetricTableMapping string `mapstructure:"metrics_table_json_mapping"` + LogTableMapping string `mapstructure:"logs_table_json_mapping"` + TraceTableMapping string `mapstructure:"traces_table_json_mapping"` + IngestionType string `mapstructure:"ingestion_type"` } // Validate checks if the exporter configuration is valid diff --git a/exporter/azuredataexplorerexporter/config_test.go b/exporter/azuredataexplorerexporter/config_test.go index 496ff9cca860..9167568179ae 100644 --- a/exporter/azuredataexplorerexporter/config_test.go +++ b/exporter/azuredataexplorerexporter/config_test.go @@ -6,11 +6,13 @@ package azuredataexplorerexporter // import "github.com/open-telemetry/opentelem import ( "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/exporter/exporterhelper" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azuredataexplorerexporter/internal/metadata" ) @@ -80,6 +82,34 @@ func TestLoadConfig(t *testing.T) { id: component.NewIDWithName(metadata.Type, "7"), errorMessage: `clusterURI config is mandatory`, }, + { + id: component.NewIDWithName(metadata.Type, "8"), + expected: &Config{ + ClusterURI: "https://CLUSTER.kusto.windows.net", + ApplicationID: "f80da32c-108c-415c-a19e-643f461a677a", + ApplicationKey: "xx-xx-xx-xx", + TenantID: "21ff9e36-fbaa-43c8-98ba-00431ea10bc3", + Database: "oteldb", + MetricTable: "OTELMetrics", + LogTable: "OTELLogs", + TraceTable: "OTELTraces", + IngestionType: managedIngestType, + TimeoutSettings: exporterhelper.TimeoutSettings{ + Timeout: 10 * time.Second, + }, + RetrySettings: exporterhelper.RetrySettings{ + Enabled: true, + InitialInterval: 10 * time.Second, + MaxInterval: 60 * time.Second, + MaxElapsedTime: 10 * time.Minute, + }, + QueueSettings: exporterhelper.QueueSettings{ + Enabled: true, + NumConsumers: 2, + QueueSize: 10, + }, + }, + }, } for _, tt := range tests { diff --git a/exporter/azuredataexplorerexporter/factory.go b/exporter/azuredataexplorerexporter/factory.go index 300ad7c06a65..ca77d82ea13e 100644 --- a/exporter/azuredataexplorerexporter/factory.go +++ b/exporter/azuredataexplorerexporter/factory.go @@ -74,7 +74,9 @@ func createMetricsExporter( set, adxCfg, adp.metricsDataPusher, - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithTimeout(adxCfg.TimeoutSettings), + exporterhelper.WithRetry(adxCfg.RetrySettings), + exporterhelper.WithQueue(adxCfg.QueueSettings), exporterhelper.WithShutdown(adp.Close)) if err != nil { @@ -104,7 +106,9 @@ func createTracesExporter( set, adxCfg, adp.tracesDataPusher, - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithTimeout(adxCfg.TimeoutSettings), + exporterhelper.WithRetry(adxCfg.RetrySettings), + exporterhelper.WithQueue(adxCfg.QueueSettings), exporterhelper.WithShutdown(adp.Close)) if err != nil { @@ -134,7 +138,9 @@ func createLogsExporter( set, adxCfg, adp.logsDataPusher, - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithTimeout(adxCfg.TimeoutSettings), + exporterhelper.WithRetry(adxCfg.RetrySettings), + exporterhelper.WithQueue(adxCfg.QueueSettings), exporterhelper.WithShutdown(adp.Close)) if err != nil { diff --git a/exporter/azuredataexplorerexporter/testdata/config.yaml b/exporter/azuredataexplorerexporter/testdata/config.yaml index 1acf2bb887ae..0fdf9fbe0a13 100644 --- a/exporter/azuredataexplorerexporter/testdata/config.yaml +++ b/exporter/azuredataexplorerexporter/testdata/config.yaml @@ -115,4 +115,34 @@ azuredataexplorer/7: # raw traces table traces_table_name: "OTELTraces" # type of ingestion managed or queued - ingestion_type: "managed" \ No newline at end of file + ingestion_type: "managed" +azuredataexplorer/8: + # Kusto cluster uri + cluster_uri: "https://CLUSTER.kusto.windows.net" + # Client Id + application_id: "f80da32c-108c-415c-a19e-643f461a677a" + # The client secret for the client + application_key: "xx-xx-xx-xx" + # The tenant + tenant_id: "21ff9e36-fbaa-43c8-98ba-00431ea10bc3" + # database for the logs + db_name: "oteldb" + # raw metric table name + metrics_table_name: "OTELMetrics" + # raw log table name + logs_table_name: "OTELLogs" + # raw traces table + traces_table_name: "OTELTraces" + # type of ingestion managed or queued + ingestion_type: "managed" + #export helper options + timeout: 10s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 10 + retry_on_failure: + enabled: true + initial_interval: 10s + max_interval: 60s + max_elapsed_time: 10m \ No newline at end of file