From a0d39d75b57d9c86e0e9b9002d1abad41acebafe Mon Sep 17 00:00:00 2001 From: Dmitry Anoshin Date: Tue, 6 Feb 2024 17:31:40 -0800 Subject: [PATCH] [chore] Adopt for the kubelet cpu utilization metrics deprecation The following metrics emitted by kubeletstats receiver were deprecated: - k8s.node.cpu.utilization - k8s.pod.cpu.utilization - container.cpu.utilization See https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/25901 for more details. Those metrics are already excluded by default in the signalfx exporter. So if we don't do anything, user will see the following confusing warnings: ``` 2024-02-06T19:21:44.773Z warn metadata/generated_metrics.go:2894 [WARNING] `container.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric container.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.773Z warn metadata/generated_metrics.go:2897 [WARNING] `k8s.node.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric k8s.node.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.773Z warn metadata/generated_metrics.go:2900 [WARNING] `k8s.pod.cpu.utilization` should not be enabled: This metric will be disabled in a future release. Use metric k8s.pod.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2894 [WARNING] `container.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric container.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2897 [WARNING] `k8s.node.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric k8s.node.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2900 [WARNING] `k8s.pod.cpu.utilization` should not be enabled: This metric will be disabled in a future release. Use metric k8s.pod.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2894 [WARNING] `container.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric container.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2897 [WARNING] `k8s.node.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric k8s.node.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2900 [WARNING] `k8s.pod.cpu.utilization` should not be enabled: This metric will be disabled in a future release. Use metric k8s.pod.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2894 [WARNING] `container.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric container.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2897 [WARNING] `k8s.node.cpu.utilization` should not be enabled: WARNING: This metric will be disabled in a future release. Use metric k8s.node.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} 2024-02-06T19:21:44.774Z warn metadata/generated_metrics.go:2900 [WARNING] `k8s.pod.cpu.utilization` should not be enabled: This metric will be disabled in a future release. Use metric k8s.pod.cpu.usage instead. {"kind": "receiver", "name": "kubeletstats", "data_type": "metrics"} ``` Potentially we couldn't just explicitly disable them in the default k8s configuration on the receiver side. The problem is that the exclusion on the signalfx exporter side potentially can be disabled by users. For those users, we actually want to show the warnings. So we need to add another converter until the metric is disabled on the kubelet receiver side by default. The converter looks at the signalfx exporter in user's config, and if metrics are not explicitly enabled on the exporter side, it disables them on the kubelet receiver side to supress the warnings. The converted required to copy `dpfilters` package from signal exporter. This will be removed along with the converter in a few releases. --- go.mod | 4 +- .../disable_kubelet_utilization_metrics.go | 145 +++++++ ...isable_kubelet_utilization_metrics_test.go | 85 ++++ .../configconverter/dpfilters/datapoint.go | 56 +++ .../configconverter/dpfilters/dimensions.go | 64 +++ .../dpfilters/dimensions_test.go | 134 ++++++ .../configconverter/dpfilters/filterset.go | 67 +++ .../dpfilters/filterset_test.go | 408 ++++++++++++++++++ .../configconverter/dpfilters/matching.go | 64 +++ .../configconverter/dpfilters/metricfilter.go | 42 ++ internal/configconverter/dpfilters/string.go | 105 +++++ .../configconverter/dpfilters/string_test.go | 162 +++++++ ...metrics_included_in_signalfx_exporter.yaml | 30 ++ .../disable_all_metrics_input.yaml | 27 ++ .../disable_all_metrics_output.yaml | 34 ++ .../do_not_change_enabled_metrics_input.yaml | 30 ++ .../do_not_change_enabled_metrics_output.yaml | 34 ++ .../no_kubeletstats_receiver.yaml | 25 ++ .../no_signalfx_exporter.yaml | 28 ++ ...y_excluded_in_signalfx_exporter_input.yaml | 29 ++ ..._excluded_in_signalfx_exporter_output.yaml | 34 ++ ...y_included_in_signalfx_exporter_input.yaml | 29 ++ ..._included_in_signalfx_exporter_output.yaml | 34 ++ .../utilization_metrics_disabled.yaml | 35 ++ internal/settings/settings.go | 1 + internal/settings/settings_test.go | 1 + 26 files changed, 1705 insertions(+), 2 deletions(-) create mode 100644 internal/configconverter/disable_kubelet_utilization_metrics.go create mode 100644 internal/configconverter/disable_kubelet_utilization_metrics_test.go create mode 100644 internal/configconverter/dpfilters/datapoint.go create mode 100644 internal/configconverter/dpfilters/dimensions.go create mode 100644 internal/configconverter/dpfilters/dimensions_test.go create mode 100644 internal/configconverter/dpfilters/filterset.go create mode 100644 internal/configconverter/dpfilters/filterset_test.go create mode 100644 internal/configconverter/dpfilters/matching.go create mode 100644 internal/configconverter/dpfilters/metricfilter.go create mode 100644 internal/configconverter/dpfilters/string.go create mode 100644 internal/configconverter/dpfilters/string_test.go create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/all_metrics_included_in_signalfx_exporter.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_input.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_output.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_input.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_output.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_kubeletstats_receiver.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_signalfx_exporter.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_input.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_output.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_input.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_output.yaml create mode 100644 internal/configconverter/testdata/disable_kubelet_utilization_metrics/utilization_metrics_disabled.yaml diff --git a/go.mod b/go.mod index e32b53ee48..987d101e35 100644 --- a/go.mod +++ b/go.mod @@ -319,7 +319,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/go-test/deep v1.1.0 // indirect - github.com/gobwas/glob v0.2.4-0.20181002190808-e7a84e9525fe // indirect + github.com/gobwas/glob v0.2.4-0.20181002190808-e7a84e9525fe github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect @@ -466,7 +466,7 @@ require ( github.com/ryanuber/go-glob v1.0.0 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect github.com/shirou/gopsutil/v3 v3.24.1 // indirect - github.com/signalfx/com_signalfx_metrics_protobuf v0.0.3 // indirect + github.com/signalfx/com_signalfx_metrics_protobuf v0.0.3 github.com/signalfx/defaults v1.2.2-0.20180531161417-70562fe60657 // indirect github.com/signalfx/gohistogram v0.0.0-20160107210732-1ccfd2ff5083 // indirect github.com/signalfx/ingest-protocols v0.2.0 // indirect diff --git a/internal/configconverter/disable_kubelet_utilization_metrics.go b/internal/configconverter/disable_kubelet_utilization_metrics.go new file mode 100644 index 0000000000..56c02e326a --- /dev/null +++ b/internal/configconverter/disable_kubelet_utilization_metrics.go @@ -0,0 +1,145 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configconverter + +import ( + "context" + "fmt" + "regexp" + + sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model" + "go.opentelemetry.io/collector/confmap" + + "github.com/signalfx/splunk-otel-collector/internal/configconverter/dpfilters" +) + +// Metrics deprecated by kubeletstats receiver that need to be disabled if not explicitly included in signalfx exporter. +const ( + k8sNodeCPUUtilization = "k8s.node.cpu.utilization" + k8sPodCPUUtilization = "k8s.pod.cpu.utilization" + containerCPUUtilization = "container.cpu.utilization" +) + +// signalfxExporterConfig is the configuration for the signalfx exporter that contains the metrics filters. +type signalfxExporterConfig struct { + ExcludeMetrics []dpfilters.MetricFilter `mapstructure:"exclude_metrics"` + IncludeMetrics []dpfilters.MetricFilter `mapstructure:"include_metrics"` +} + +// DisableKubeletUtilizationMetrics is a MapConverter that disables the following deprecated metrics: +// - `k8s.node.cpu.utilization` +// - `k8s.pod.cpu.utilization` +// - `container.cpu.utilization` +// The converter disables the metrics at the receiver level to avoid showing users a warning message because +// they are excluded in signalfx exporter by default. +// We don't disable them in case if users explicitly include them in signalfx exporter. +type DisableKubeletUtilizationMetrics struct{} + +func (DisableKubeletUtilizationMetrics) Convert(_ context.Context, cfgMap *confmap.Conf) error { + if cfgMap == nil { + return fmt.Errorf("cannot DisableKubeletUtilizationMetrics on nil *confmap.Conf") + } + + receivers, err := cfgMap.Sub("receivers") + if err != nil { + return nil // Ignore invalid config. Rely on the config validation to catch this. + } + kubeletReceiverConfigs := map[string]map[string]any{} + for receiverName, receiverCfg := range receivers.ToStringMap() { + if regexp.MustCompile("kubeletstats(/\\w+)?").MatchString(receiverName) { + if v, ok := receiverCfg.(map[string]any); ok { + kubeletReceiverConfigs[receiverName] = v + } + } + } + + exporters, err := cfgMap.Sub("exporters") + if err != nil { + return nil // Ignore invalid config. Rely on the config validation to catch this. + } + sfxExporterConfigs := map[string]map[string]any{} + for exporterName, exporterCfg := range exporters.ToStringMap() { + if regexp.MustCompile("signalfx(/\\w+)?").MatchString(exporterName) { + if v, ok := exporterCfg.(map[string]any); ok { + sfxExporterConfigs[exporterName] = v + } + } + } + + // If there is no signalfx exporter or kubeletstats receiver, there is nothing to do. + if len(kubeletReceiverConfigs) == 0 || len(sfxExporterConfigs) == 0 { + return nil + } + + disableMetrics := map[string]bool{ + k8sNodeCPUUtilization: true, + k8sPodCPUUtilization: true, + containerCPUUtilization: true, + } + + // Check if the metrics are explicitly included in signalfx exporter. + // If they are not included, we will disable them in kubeletstats receiver. + for _, cm := range sfxExporterConfigs { + cfg := signalfxExporterConfig{} + err = confmap.NewFromStringMap(cm).Unmarshal(&cfg, confmap.WithIgnoreUnused()) + if err != nil { + return nil // Ignore invalid config. Rely on the config validation to catch this. + } + if len(cfg.ExcludeMetrics) == 0 { + // Apply default excluded metrics if not explicitly set. + // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/f2d8efe507083b0f38b6567f8dba3f37053bfa86/exporter/signalfxexporter/internal/translation/default_metrics.go#L133 + cfg.ExcludeMetrics = []dpfilters.MetricFilter{ + {MetricNames: []string{"/^(?i:(container)|(k8s\\.node)|(k8s\\.pod))\\.cpu\\.utilization$/"}}, + } + } + + filter, err := dpfilters.NewFilterSet(cfg.ExcludeMetrics, cfg.IncludeMetrics) + if err != nil { + return nil // Ignore invalid config. Rely on the config validation to catch this. + } + for metricName := range disableMetrics { + if !filter.Matches(&sfxpb.DataPoint{Metric: metricName}) { + disableMetrics[metricName] = false + } + } + } + + // Disable the metrics in kubeletstats receiver. + for receiverName, cfg := range kubeletReceiverConfigs { + metricsCfg := map[string]any{} + if cfg["metrics"] != nil { + if v, ok := cfg["metrics"].(map[string]any); ok { + metricsCfg = v + } + } + if _, ok := metricsCfg[k8sNodeCPUUtilization]; !ok && disableMetrics[k8sNodeCPUUtilization] { + metricsCfg[k8sNodeCPUUtilization] = map[string]any{"enabled": false} + } + if _, ok := metricsCfg[k8sPodCPUUtilization]; !ok && disableMetrics[k8sPodCPUUtilization] { + metricsCfg[k8sPodCPUUtilization] = map[string]any{"enabled": false} + } + if _, ok := metricsCfg[containerCPUUtilization]; !ok && disableMetrics[containerCPUUtilization] { + metricsCfg[containerCPUUtilization] = map[string]any{"enabled": false} + } + metricsCfgKey := fmt.Sprintf("receivers::%s::metrics", receiverName) + if len(metricsCfg) > 0 { + if err = cfgMap.Merge(confmap.NewFromStringMap(map[string]any{metricsCfgKey: metricsCfg})); err != nil { + return err + } + } + } + + return nil +} diff --git a/internal/configconverter/disable_kubelet_utilization_metrics_test.go b/internal/configconverter/disable_kubelet_utilization_metrics_test.go new file mode 100644 index 0000000000..6cf70316da --- /dev/null +++ b/internal/configconverter/disable_kubelet_utilization_metrics_test.go @@ -0,0 +1,85 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configconverter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap/confmaptest" +) + +func TestDisableKubeletUtilizationMetrics(t *testing.T) { + tests := []struct { + name string + input string + wantOutput string + }{ + { + name: "no_kubeletstats_receiver", + input: "testdata/disable_kubelet_utilization_metrics/no_kubeletstats_receiver.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/no_kubeletstats_receiver.yaml", + }, + { + name: "no_signalfx_exporter", + input: "testdata/disable_kubelet_utilization_metrics/no_signalfx_exporter.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/no_signalfx_exporter.yaml", + }, + { + name: "disable_all_metrics", + input: "testdata/disable_kubelet_utilization_metrics/disable_all_metrics_input.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/disable_all_metrics_output.yaml", + }, + { + name: "do_not_change_enabled_metrics", + input: "testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_input.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_output.yaml", + }, + { + name: "all_metrics_included_in_signalfx_exporter", + input: "testdata/disable_kubelet_utilization_metrics/all_metrics_included_in_signalfx_exporter.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/all_metrics_included_in_signalfx_exporter.yaml", + }, + { + name: "partially_excluded_in_signalfx_exporter", + input: "testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_input.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_output.yaml", + }, + { + name: "partially_included_in_signalfx_exporter", + input: "testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_input.yaml", + wantOutput: "testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_output.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectedCfgMap, err := confmaptest.LoadConf(tt.wantOutput) + require.NoError(t, err) + require.NotNil(t, expectedCfgMap) + + cfgMap, err := confmaptest.LoadConf(tt.input) + require.NoError(t, err) + require.NotNil(t, cfgMap) + + err = DisableKubeletUtilizationMetrics{}.Convert(context.Background(), cfgMap) + require.NoError(t, err) + + assert.Equal(t, expectedCfgMap, cfgMap) + }) + } + +} diff --git a/internal/configconverter/dpfilters/datapoint.go b/internal/configconverter/dpfilters/datapoint.go new file mode 100644 index 0000000000..4ae5900148 --- /dev/null +++ b/internal/configconverter/dpfilters/datapoint.go @@ -0,0 +1,56 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters" + +import ( + "errors" + + sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model" +) + +type dataPointFilter struct { + metricFilter *StringFilter + dimensionsFilter *dimensionsFilter +} + +// newDataPointFilter returns a new dataPointFilter filter with the given configuration. +func newDataPointFilter(metricNames []string, dimSet map[string][]string) (*dataPointFilter, error) { + var metricFilter *StringFilter + if len(metricNames) > 0 { + var err error + metricFilter, err = NewStringFilter(metricNames) + if err != nil { + return nil, err + } + } + + var dimensionsFilter *dimensionsFilter + if len(dimSet) > 0 { + var err error + dimensionsFilter, err = newDimensionsFilter(dimSet) + if err != nil { + return nil, err + } + } + + if metricFilter == nil && dimensionsFilter == nil { + return nil, errors.New("metric filter must have at least one metric or dimension defined on it") + } + + return &dataPointFilter{ + metricFilter: metricFilter, + dimensionsFilter: dimensionsFilter, + }, nil +} + +// Matches tests a datapoint to see whether it is excluded by this +func (f *dataPointFilter) Matches(dp *sfxpb.DataPoint) bool { + metricNameMatched := f.metricFilter == nil || f.metricFilter.Matches(dp.Metric) + if metricNameMatched { + return f.dimensionsFilter == nil || f.dimensionsFilter.Matches(dp.Dimensions) + } + return false + +} diff --git a/internal/configconverter/dpfilters/dimensions.go b/internal/configconverter/dpfilters/dimensions.go new file mode 100644 index 0000000000..38267efe04 --- /dev/null +++ b/internal/configconverter/dpfilters/dimensions.go @@ -0,0 +1,64 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters" + +import ( + "errors" + + sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model" +) + +type dimensionsFilter struct { + filterMap map[string]*StringFilter +} + +// newDimensionsFilter returns a filter that matches against a +// sfxpb.Dimension slice. The filter will return false if there's +// at least one dimension in the slice that fails to match. In case` +// there are no filters for any of the dimension keys in the slice, +// the filter will return false. +func newDimensionsFilter(m map[string][]string) (*dimensionsFilter, error) { + filterMap := map[string]*StringFilter{} + for k := range m { + if len(m[k]) == 0 { + return nil, errors.New("string map value in filter cannot be empty") + } + + var err error + filterMap[k], err = NewStringFilter(m[k]) + if err != nil { + return nil, err + } + } + + return &dimensionsFilter{ + filterMap: filterMap, + }, nil +} + +func (f *dimensionsFilter) Matches(dimensions []*sfxpb.Dimension) bool { + if len(dimensions) == 0 { + return false + } + + var atLeastOneMatchedDimension bool + for _, dim := range dimensions { + dimF := f.filterMap[dim.Key] + // Skip if there are no filters associated with current dimension key. + if dimF == nil { + continue + } + + if !dimF.Matches(dim.Value) { + return false + } + + if !atLeastOneMatchedDimension { + atLeastOneMatchedDimension = true + } + } + + return atLeastOneMatchedDimension +} diff --git a/internal/configconverter/dpfilters/dimensions_test.go b/internal/configconverter/dpfilters/dimensions_test.go new file mode 100644 index 0000000000..cc3cc0b36b --- /dev/null +++ b/internal/configconverter/dpfilters/dimensions_test.go @@ -0,0 +1,134 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters + +import ( + "testing" + + sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model" + "github.com/stretchr/testify/require" +) + +func TestDimensionsFilter(t *testing.T) { + tests := []struct { + name string + filter map[string][]string + input []*sfxpb.Dimension + shouldMatch bool + shouldError bool + }{ + { + name: "Empty filter does not empty slice of dimensions", + filter: map[string][]string{}, + input: []*sfxpb.Dimension{}, + shouldMatch: false, + }, + { + name: "Non-empty filter does not match empty slice of dimensions", + filter: map[string][]string{ + "app": {"test"}, + }, + input: []*sfxpb.Dimension{}, + shouldMatch: false, + }, + { + name: "Filter does not match different dimension", + filter: map[string][]string{ + "app": {"test"}, + }, + input: []*sfxpb.Dimension{ + { + Key: "version", + Value: "latest", + }, + }, + shouldMatch: false, + }, + { + name: "Filter matches on exact match of a dimension", + filter: map[string][]string{ + "app": {"test"}, + "version": {"*"}, + }, + input: []*sfxpb.Dimension{ + { + Key: "app", + Value: "test", + }, + }, + shouldMatch: true, + }, + { + name: "Filter matches on exact match with multiple dimensions in input slice", + filter: map[string][]string{ + "app": {"test"}, + }, + input: []*sfxpb.Dimension{ + { + Key: "app", + Value: "test", + }, + { + Key: "version", + Value: "2.0", + }, + }, + shouldMatch: true, + }, + { + name: "Filter matches on regex with multiple dimensions in input slice", + filter: map[string][]string{ + "version": {`/\d+\.\d+/`}, + }, + input: []*sfxpb.Dimension{ + { + Key: "app", + Value: "test", + }, + { + Key: "version", + Value: "2.0", + }, + }, + shouldMatch: true, + }, + { + name: "Filter does not match on regex", + filter: map[string][]string{ + "version": {`/\d+\.\d+/`}, + }, + input: []*sfxpb.Dimension{ + { + Key: "app", + Value: "test", + }, + { + Key: "version", + Value: "bad", + }, + }, + shouldMatch: false, + }, + { + name: "Error creating filter with no dimension values", + filter: map[string][]string{ + "version": {}, + }, + shouldError: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f, err := newDimensionsFilter(test.filter) + if test.shouldError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Equal(t, test.shouldMatch, f.Matches(test.input)) + }) + } +} diff --git a/internal/configconverter/dpfilters/filterset.go b/internal/configconverter/dpfilters/filterset.go new file mode 100644 index 0000000000..216d1210ab --- /dev/null +++ b/internal/configconverter/dpfilters/filterset.go @@ -0,0 +1,67 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters" + +import sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model" + +// FilterSet is a collection of datapont filters, any one of which must match +// for a datapoint to be matched. +type FilterSet struct { + excludeFilters []*dataPointFilter + includeFilters []*dataPointFilter +} + +// Matches sends a datapoint through each of the filters in the set and returns +// true if at least one of them matches the datapoint. +func (fs *FilterSet) Matches(dp *sfxpb.DataPoint) bool { + for _, ex := range fs.excludeFilters { + if ex.Matches(dp) { + // If we match an exclusionary filter, run through each inclusion + // filter and see if anything includes the metrics. + for _, in := range fs.includeFilters { + if in.Matches(dp) { + return false + } + } + return true + } + } + return false +} + +func NewFilterSet(excludes []MetricFilter, includes []MetricFilter) (*FilterSet, error) { + excludeSet, err := getDataPointFilters(excludes) + if err != nil { + return nil, err + } + + includeSet, err := getDataPointFilters(includes) + if err != nil { + return nil, err + } + + return &FilterSet{ + excludeFilters: excludeSet, + includeFilters: includeSet, + }, nil +} + +func getDataPointFilters(metricFilters []MetricFilter) ([]*dataPointFilter, error) { + out := make([]*dataPointFilter, len(metricFilters)) + for i, f := range metricFilters { + dimSet, err := f.normalize() + if err != nil { + return nil, err + } + + dpf, err := newDataPointFilter(f.MetricNames, dimSet) + if err != nil { + return nil, err + } + + out[i] = dpf + } + return out, nil +} diff --git a/internal/configconverter/dpfilters/filterset_test.go b/internal/configconverter/dpfilters/filterset_test.go new file mode 100644 index 0000000000..7e8f7e8798 --- /dev/null +++ b/internal/configconverter/dpfilters/filterset_test.go @@ -0,0 +1,408 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters + +import ( + "testing" + + sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model" + "github.com/stretchr/testify/require" +) + +func TestFilterSet(t *testing.T) { + tests := []struct { + name string + excludes []MetricFilter + includes []MetricFilter + expectedMatches []*sfxpb.DataPoint + expectedNonMatches []*sfxpb.DataPoint + wantErr bool + wantErrMsg string + }{ + { + name: "Match based on simple metric name as string", + excludes: []MetricFilter{{MetricName: "cpu.utilization"}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "memory.utilization", + }, + }, + }, + { + name: "Match based on simple metric name", + excludes: []MetricFilter{{MetricNames: []string{"cpu.utilization"}}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "memory.utilization", + }, + }, + }, + { + name: "Match based on multiple metric names", + excludes: []MetricFilter{{MetricNames: []string{"cpu.utilization", "memory.utilization"}}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + { + Metric: "memory.utilization", + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + }, + }, + }, + { + name: "Match based on regex metric name", + excludes: []MetricFilter{{MetricNames: []string{`/cpu\..*/`}}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + }, + }, + }, + { + name: "Match based on glob metric name", + excludes: []MetricFilter{{MetricNames: []string{`cpu.util*`, "memor*"}}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + { + Metric: "memory.utilization", + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + }, + }, + }, + { + name: "Match based on dimension name as string", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": "PO", + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "PO"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + }, + { + name: "Match based on dimension name", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{"PO"}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "PO"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + }, + { + name: "Match based on dimension name regex", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{`/^[A-Z][A-Z]$/`}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "PO"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + }, + { + name: "Match based on dimension presence", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{`/.+/`}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "host", Value: "localhost"}}, + }, + }, + }, + { + name: "Match based on dimension name glob", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{`*O*`}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "POD"}}, + }, + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "POD123"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + }, + { + name: "Match based on conjunction of both dimensions and metric name", + excludes: []MetricFilter{{ + MetricNames: []string{"*.utilization"}, + Dimensions: map[string]any{ + "container_name": []any{"test"}, + }, + }}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "not matching"}}, + }, { + Metric: "disk.usage", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + }, + { + name: "Doesn't match if no dimension filter specified", + excludes: []MetricFilter{{MetricNames: []string{"cpu.utilization"}}}, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "disk.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "test"}}, + }, + }, + }, + { + name: "Doesn't match if no metric name filter specified", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{"mycontainer"}, + }}}, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + }, + }, + { + name: "Doesn't match metric when no (matching) dimensions exist", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "host": []any{"localhost"}, + "system": []any{"r4"}, + }}}, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{ + {Key: "Host", Value: "localhost"}, + }, + }, + }, + }, + { + name: "Matches on at least one dimension", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "host": []any{"localhost"}, + "system": []any{"r4"}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{ + {Key: "host", Value: "localhost"}, + }, + }, + }, + }, + { + name: "Matches against all dimension pairs", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "host": []any{"localhost"}, + "system": []any{"r4"}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{ + {Key: "host", Value: "localhost"}, + {Key: "system", Value: "r4"}, + }, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{ + {Key: "host", Value: "localhost"}, + {Key: "system", Value: "r3"}, + }, + }, + }, + }, + { + name: "Negated dim values take precedent", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{"*", "!pause", "!/.*idle/"}, + }}}, + expectedMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "mycontainer"}}, + }, + }, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "pause"}}, + }, + { + Metric: "cpu.utilization", + Dimensions: []*sfxpb.Dimension{{Key: "container_name", Value: "is_idle"}}, + }, + }, + }, + { + name: "Error creating exclude empty filter", + excludes: []MetricFilter{{}}, + wantErr: true, + wantErrMsg: "metric filter must have at least one metric or dimension defined on it", + }, + { + name: "Error creating include empty filter", + includes: []MetricFilter{{}}, + wantErr: true, + wantErrMsg: "metric filter must have at least one metric or dimension defined on it", + }, + { + name: "Error creating filter with empty dimension list", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "dim": []any{}, + }}}, + wantErr: true, + wantErrMsg: "string map value in filter cannot be empty", + }, + { + name: "Error creating filter with invalid glob", + excludes: []MetricFilter{{MetricNames: []string{"cpu.*["}}}, + wantErr: true, + wantErrMsg: "unexpected end of input", + }, + { + name: "Error creating filter with invalid glob in dimensions", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "container_name": []any{"cpu.*["}, + }}}, + wantErr: true, + wantErrMsg: "unexpected end of input", + }, + { + name: "Error on invalid dimensions input", + excludes: []MetricFilter{{ + Dimensions: map[string]any{ + "host": 1, + }}}, + wantErr: true, + wantErrMsg: "1 should be either a string or string list", + }, + { + name: "Match in include filters correctly overrides exclude", + excludes: []MetricFilter{{MetricNames: []string{"cpu.utilization"}}}, + includes: []MetricFilter{{MetricNames: []string{"cpu.utilization"}}}, + expectedNonMatches: []*sfxpb.DataPoint{ + { + Metric: "cpu.utilization", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f, err := NewFilterSet(test.excludes, test.includes) + if test.wantErr { + require.EqualError(t, err, test.wantErrMsg) + require.Nil(t, f) + return + } + require.NoError(t, err) + + for _, metric := range test.expectedMatches { + require.True(t, f.Matches(metric)) + } + + for _, metric := range test.expectedNonMatches { + require.False(t, f.Matches(metric)) + } + }) + } +} diff --git a/internal/configconverter/dpfilters/matching.go b/internal/configconverter/dpfilters/matching.go new file mode 100644 index 0000000000..31a4cbf0c5 --- /dev/null +++ b/internal/configconverter/dpfilters/matching.go @@ -0,0 +1,64 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters" + +import ( + "regexp" + "strings" + + "github.com/gobwas/glob" +) + +// Contains all of the logic for glob and regex based filtering. + +func isGlobbed(s string) bool { + return strings.ContainsAny(s, "*?[]{}!") +} + +func isRegex(s string) bool { + return len(s) > 2 && s[0] == '/' && s[len(s)-1] == '/' +} + +// remove the bracketing slashes for a regex. +func stripSlashes(s string) string { + return s[1 : len(s)-1] +} + +// stripNegation checks if a string is prefixed with "!" +// and will returned the stripped string and true if so +// else, return original value and false. +func stripNegation(value string) (string, bool) { + if strings.HasPrefix(value, "!") { + return value[1:], true + } + return value, false +} + +type matcher interface { + // Returns whether the string matched and whether it was a negated match. + Matches(s string) (bool, bool) +} + +type regexMatcher struct { + re *regexp.Regexp + negated bool +} + +var _ matcher = (*regexMatcher)(nil) + +func (m *regexMatcher) Matches(s string) (bool, bool) { + return m.re.MatchString(s), m.negated +} + +type globMatcher struct { + glob glob.Glob + negated bool +} + +var _ matcher = &globMatcher{} + +func (m *globMatcher) Matches(s string) (bool, bool) { + return m.glob.Match(s), m.negated +} diff --git a/internal/configconverter/dpfilters/metricfilter.go b/internal/configconverter/dpfilters/metricfilter.go new file mode 100644 index 0000000000..91c3ac1752 --- /dev/null +++ b/internal/configconverter/dpfilters/metricfilter.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters" + +import "fmt" + +type MetricFilter struct { + // A single metric name to match against. + MetricName string `mapstructure:"metric_name"` + // A list of metric names to match against. + MetricNames []string `mapstructure:"metric_names"` + // A map of dimension key/values to match against. All key/values must + // match a datapoint for it to be matched. The map values can be either + // a single string or a list of strings. + Dimensions map[string]any `mapstructure:"dimensions"` +} + +func (mf *MetricFilter) normalize() (map[string][]string, error) { + if mf.MetricName != "" { + mf.MetricNames = append(mf.MetricNames, mf.MetricName) + } + + dimSet := map[string][]string{} + for k, v := range mf.Dimensions { + switch s := v.(type) { + case []any: + var newSet []string + for _, iv := range s { + newSet = append(newSet, fmt.Sprintf("%v", iv)) + } + dimSet[k] = newSet + case string: + dimSet[k] = []string{s} + default: + return nil, fmt.Errorf("%v should be either a string or string list", v) + } + } + + return dimSet, nil +} diff --git a/internal/configconverter/dpfilters/string.go b/internal/configconverter/dpfilters/string.go new file mode 100644 index 0000000000..9a06453da7 --- /dev/null +++ b/internal/configconverter/dpfilters/string.go @@ -0,0 +1,105 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters" + +import ( + "regexp" + + "github.com/gobwas/glob" +) + +// StringFilter will match if any one of the given strings is a match. +type StringFilter struct { + staticSet map[string]bool + regexps []regexMatcher + globs []globMatcher + anyStaticNegated bool +} + +// NewStringFilter returns a filter that can match against the provided items. +func NewStringFilter(items []string) (*StringFilter, error) { + staticSet := make(map[string]bool) + var regexps []regexMatcher + var globs []globMatcher + + anyStaticNegated := false + for _, i := range items { + m, negated := stripNegation(i) + switch { + case isRegex(m): + var re *regexp.Regexp + var err error + + reText := stripSlashes(m) + re, err = regexp.Compile(reText) + + if err != nil { + return nil, err + } + + regexps = append(regexps, regexMatcher{re: re, negated: negated}) + case isGlobbed(m): + g, err := glob.Compile(m) + if err != nil { + return nil, err + } + + globs = append(globs, globMatcher{glob: g, negated: negated}) + default: + staticSet[m] = negated + if negated { + anyStaticNegated = true + } + } + } + + return &StringFilter{ + staticSet: staticSet, + regexps: regexps, + globs: globs, + anyStaticNegated: anyStaticNegated, + }, nil +} + +// Matches if s is positively matched by the filter items OR +// if it is positively matched by a non-glob/regex pattern exactly +// and is negated as well. See the unit tests for examples. +func (f *StringFilter) Matches(s string) bool { + if f == nil { + return true + } + negated, matched := f.staticSet[s] + // If a metric is negated and it matched it won't match anything else by + // definition. + if matched && negated { + return false + } + + for _, reMatch := range f.regexps { + reMatched, negated := reMatch.Matches(s) + if reMatched && negated { + return false + } + matched = matched || reMatched + } + + for _, globMatcher := range f.globs { + globMatched, negated := globMatcher.Matches(s) + if globMatched && negated { + return false + } + matched = matched || globMatched + } + return matched +} + +func (f *StringFilter) UnmarshalText(in []byte) error { + sf, err := NewStringFilter([]string{string(in)}) + if err != nil { + return err + } + *f = *sf + return nil +} diff --git a/internal/configconverter/dpfilters/string_test.go b/internal/configconverter/dpfilters/string_test.go new file mode 100644 index 0000000000..ad74b3dd0d --- /dev/null +++ b/internal/configconverter/dpfilters/string_test.go @@ -0,0 +1,162 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// Temporary copied from the SignalFx exporter to be used in DisableKubeletUtilizationMetrics converter. + +package dpfilters + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringFilter(t *testing.T) { + tests := []struct { + name string + filter []string + inputs []string + shouldMatch []bool + shouldError bool + }{ + { + filter: []string{}, + inputs: []string{"process_", "", "asdf"}, + shouldMatch: []bool{false, false, false}, + }, + { + filter: []string{ + "*", + }, + inputs: []string{"app", "asdf", "", "*"}, + shouldMatch: []bool{true, true, true, true}, + }, + { + filter: []string{ + "!app", + }, + inputs: []string{"app", "other"}, + shouldMatch: []bool{false, false}, + }, + { + filter: []string{ + "app", + "!app", + }, + inputs: []string{"app", "other"}, + shouldMatch: []bool{false, false}, + }, + { + filter: []string{ + "other", + "!app", + }, + inputs: []string{"other", "something", "app"}, + shouldMatch: []bool{true, false, false}, + }, + { + filter: []string{ + "/^process_/", + "/^node_/", + }, + inputs: []string{"process_", "node_", "process_asdf", "other"}, + shouldMatch: []bool{true, true, true, false}, + }, + { + filter: []string{ + "!/^process_/", + }, + inputs: []string{"process_", "other"}, + shouldMatch: []bool{false, false}, + }, + { + filter: []string{ + "app", + "!/^process_/", + "process_", + }, + inputs: []string{"other", "app", "process_cpu", "process_"}, + shouldMatch: []bool{false, true, false, false}, + }, + { + filter: []string{ + "asdfdfasdf", + "/^node_/", + }, + inputs: []string{"node_test"}, + shouldMatch: []bool{true}, + }, + { + filter: []string{ + "process_*", + "!process_cpu", + }, + inputs: []string{"process_mem", "process_cpu", "asdf"}, + shouldMatch: []bool{true, false, false}, + }, + { + filter: []string{ + "*", + "!process_cpu", + }, + inputs: []string{"process_mem", "process_cpu", "asdf"}, + shouldMatch: []bool{true, false, true}, + }, + { + filter: []string{ + "metric_?", + "!metric_a", + "!metric_b", + "random", + }, + inputs: []string{"metric_a", "metric_b", "metric_c", "asdf", "random"}, + shouldMatch: []bool{false, false, true, false, true}, + }, + { + filter: []string{ + "!process_cpu", + // Order doesn't matter + "*", + }, + inputs: []string{"process_mem", "process_cpu", "asdf"}, + shouldMatch: []bool{true, false, true}, + }, + { + filter: []string{ + "/a.*/", + "!/.*z/", + "b", + // Static match should not override the negated regex above + "alz", + }, + inputs: []string{"", "asdf", "asdz", "b", "wrong", "alz"}, + shouldMatch: []bool{false, true, false, true, false, false}, + }, + { + filter: []string{"!memory*"}, + inputs: []string{"cpu.utilization", "memory.utilization"}, + shouldMatch: []bool{false, false}, + }, + { + filter: []string{"/!memor*(/"}, + shouldError: true, + }, + { + filter: nil, + inputs: []string{"cpu.utilization", "memory.utilization"}, + shouldMatch: []bool{false, false}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f, err := NewStringFilter(test.filter) + if test.shouldError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + for i := range test.inputs { + assert.Equal(t, test.shouldMatch[i], f.Matches(test.inputs[i])) + } + }) + } +} diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/all_metrics_included_in_signalfx_exporter.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/all_metrics_included_in_signalfx_exporter.yaml new file mode 100644 index 0000000000..f01060d60b --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/all_metrics_included_in_signalfx_exporter.yaml @@ -0,0 +1,30 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM + include_metrics: + - metric_names: + - /^(?i:(container)|(k8s\.node)|(k8s\.pod))\.cpu\.utilization$/ +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_input.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_input.yaml new file mode 100644 index 0000000000..8b54e6307e --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_input.yaml @@ -0,0 +1,27 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_output.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_output.yaml new file mode 100644 index 0000000000..890a1b35de --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/disable_all_metrics_output.yaml @@ -0,0 +1,34 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s + metrics: + k8s.node.cpu.utilization: + enabled: false + k8s.pod.cpu.utilization: + enabled: false + container.cpu.utilization: + enabled: false +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_input.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_input.yaml new file mode 100644 index 0000000000..547a5fa47c --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_input.yaml @@ -0,0 +1,30 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s + metrics: + k8s.pod.cpu.utilization: + enabled: true +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_output.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_output.yaml new file mode 100644 index 0000000000..2b01b29472 --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/do_not_change_enabled_metrics_output.yaml @@ -0,0 +1,34 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s + metrics: + k8s.node.cpu.utilization: + enabled: false + k8s.pod.cpu.utilization: + enabled: true + container.cpu.utilization: + enabled: false +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_kubeletstats_receiver.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_kubeletstats_receiver.yaml new file mode 100644 index 0000000000..6774f36ee5 --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_kubeletstats_receiver.yaml @@ -0,0 +1,25 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + debug: + verbosity: normal + sampling_initial: 2 + sampling_thereafter: 500 +service: + pipelines: + metrics: + receivers: + - hostmetrics + processors: + - memory_limiter + exporters: + - logging diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_signalfx_exporter.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_signalfx_exporter.yaml new file mode 100644 index 0000000000..757c0938db --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/no_signalfx_exporter.yaml @@ -0,0 +1,28 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + debug: + verbosity: normal + sampling_initial: 2 + sampling_thereafter: 500 +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - logging diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_input.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_input.yaml new file mode 100644 index 0000000000..49f5864e52 --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_input.yaml @@ -0,0 +1,29 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM + exclude_metrics: + - metric_name: /^k8s\.(node|pod)\.cpu\.utilization$/ +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_output.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_output.yaml new file mode 100644 index 0000000000..df88ed3ac0 --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_excluded_in_signalfx_exporter_output.yaml @@ -0,0 +1,34 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s + metrics: + k8s.node.cpu.utilization: + enabled: false + k8s.pod.cpu.utilization: + enabled: false +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM + exclude_metrics: + - metric_name: /^k8s\.(node|pod)\.cpu\.utilization$/ +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - signalfx diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_input.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_input.yaml new file mode 100644 index 0000000000..5813c4b56d --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_input.yaml @@ -0,0 +1,29 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats/bar: + collection_interval: 2s +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx/foo: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM + include_metrics: + - metric_name: container.cpu.utilization +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats/bar + processors: + - memory_limiter + exporters: + - signalfx/foo diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_output.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_output.yaml new file mode 100644 index 0000000000..c7be543cbd --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/partially_included_in_signalfx_exporter_output.yaml @@ -0,0 +1,34 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats/bar: + collection_interval: 2s + metrics: + k8s.node.cpu.utilization: + enabled: false + k8s.pod.cpu.utilization: + enabled: false +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + signalfx/foo: + access_token: MY_SIGNALFX_API_TOKEN + realm: MY_SIGNALFX_REALM + include_metrics: + - metric_name: container.cpu.utilization +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats/bar + processors: + - memory_limiter + exporters: + - signalfx/foo diff --git a/internal/configconverter/testdata/disable_kubelet_utilization_metrics/utilization_metrics_disabled.yaml b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/utilization_metrics_disabled.yaml new file mode 100644 index 0000000000..2f09ebcbae --- /dev/null +++ b/internal/configconverter/testdata/disable_kubelet_utilization_metrics/utilization_metrics_disabled.yaml @@ -0,0 +1,35 @@ +receivers: + hostmetrics: + collection_interval: 1s + scrapers: + cpu: + kubeletstats: + collection_interval: 2s + metrics: + k8s.node.cpu.utilization: + enabled: false + k8s.pod.cpu.utilization: + enabled: false + container.cpu.utilization: + enabled: false +processors: + memory_limiter: + check_interval: 1s + limit_mib: 4000 + spike_limit_mib: 800 + ballast_size_mib: 64 +exporters: + debug: + verbosity: normal + sampling_initial: 2 + sampling_thereafter: 500 +service: + pipelines: + metrics: + receivers: + - hostmetrics + - kubeletstats + processors: + - memory_limiter + exporters: + - logging diff --git a/internal/settings/settings.go b/internal/settings/settings.go index e7aae81dbb..70d2f42dbc 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -222,6 +222,7 @@ func (s *Settings) ConfMapConverters() []confmap.Converter { configconverter.RenameK8sTagger{}, configconverter.NormalizeGcp{}, configconverter.LogLevelToVerbosity{}, + configconverter.DisableKubeletUtilizationMetrics{}, ) } return confMapConverters diff --git a/internal/settings/settings_test.go b/internal/settings/settings_test.go index ecda8c7695..ad85b67b94 100644 --- a/internal/settings/settings_test.go +++ b/internal/settings/settings_test.go @@ -180,6 +180,7 @@ func TestNewSettingsConvertConfig(t *testing.T) { configconverter.RenameK8sTagger{}, configconverter.NormalizeGcp{}, configconverter.LogLevelToVerbosity{}, + configconverter.DisableKubeletUtilizationMetrics{}, }, settings.ConfMapConverters()) require.Equal(t, []string{"--feature-gates", "foo", "--feature-gates", "-bar", "--feature-gates", "-telemetry.useOtelForInternalMetrics"}, settings.ColCoreArgs()) }