From ab628320baed10aefa255a6cf9eadde25ab156f0 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Mon, 11 Oct 2021 10:15:35 -0600 Subject: [PATCH 1/5] fix: trim whitespace on ethtool input plugin The upstream vmxnet3 driver declares the network driver stats descriptions with leading and trailing whitespace. This was done to help with the ethtool output on the CLI. The ethtool plugin uses a library that polls the driver's stats directly and therefore this ugly whitespace will show up in Telegraf output as well. --- plugins/inputs/ethtool/ethtool_linux.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/ethtool/ethtool_linux.go b/plugins/inputs/ethtool/ethtool_linux.go index 6c0116e6e8089..01e50a671c1b6 100644 --- a/plugins/inputs/ethtool/ethtool_linux.go +++ b/plugins/inputs/ethtool/ethtool_linux.go @@ -5,6 +5,7 @@ package ethtool import ( "net" + "strings" "sync" "github.com/pkg/errors" @@ -81,7 +82,7 @@ func (e *Ethtool) gatherEthtoolStats(iface net.Interface, acc telegraf.Accumulat fields[fieldInterfaceUp] = e.interfaceUp(iface) for k, v := range stats { - fields[k] = v + fields[strings.TrimSpace(k)] = v } acc.AddFields(pluginName, fields, tags) From 3fe68600d1debce5fdb7538b70d980ee02a8151c Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Tue, 12 Oct 2021 16:27:04 -0600 Subject: [PATCH 2/5] create normalize keys option + tests --- plugins/inputs/ethtool/README.md | 8 +++ plugins/inputs/ethtool/ethtool.go | 11 +++ plugins/inputs/ethtool/ethtool_linux.go | 28 +++++++- plugins/inputs/ethtool/ethtool_test.go | 91 +++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/ethtool/README.md b/plugins/inputs/ethtool/README.md index 1b36001d9b74c..6d5b36d1b5e31 100644 --- a/plugins/inputs/ethtool/README.md +++ b/plugins/inputs/ethtool/README.md @@ -12,6 +12,14 @@ The ethtool input plugin pulls ethernet device stats. Fields pulled will depend ## List of interfaces to ignore when pulling metrics. # interface_exclude = ["eth1"] + + ## Some drivers declare statistics with extra whitespace, different spacing, + ## and mix cases. This list, when enabled, can be used to clean the keys. + ## Here are the current possible normalizations: + ## * trim: removes leading and trailing whitespace + ## * lower: changes all capitalized letters to lowercase + ## * spaces: replaces spaces with underscores + # normalize_keys = ["trim", "lower", "spaces"] ``` Interfaces can be included or ignored using: diff --git a/plugins/inputs/ethtool/ethtool.go b/plugins/inputs/ethtool/ethtool.go index 0978bef837383..fa54f1e472bf0 100644 --- a/plugins/inputs/ethtool/ethtool.go +++ b/plugins/inputs/ethtool/ethtool.go @@ -20,6 +20,9 @@ type Ethtool struct { // This is the list of interface names to ignore InterfaceExclude []string `toml:"interface_exclude"` + // Normalization on the key names + NormalizeKeys []string `toml:"normalize_keys"` + Log telegraf.Logger `toml:"-"` // the ethtool command @@ -38,6 +41,14 @@ const ( ## List of interfaces to ignore when pulling metrics. # interface_exclude = ["eth1"] + + ## Some drivers declare statistics with extra whitespace, different spacing, + ## and mix cases. This list, when enabled, can be used to clean the keys. + ## Here are the current possible normalizations: + ## * trim: removes leading and trailing whitespace + ## * lower: changes all capitalized letters to lowercase + ## * spaces: replaces spaces with underscores + # normalize_keys = ["trim", "lower", "spaces"] ` ) diff --git a/plugins/inputs/ethtool/ethtool_linux.go b/plugins/inputs/ethtool/ethtool_linux.go index 01e50a671c1b6..a3814043eff9c 100644 --- a/plugins/inputs/ethtool/ethtool_linux.go +++ b/plugins/inputs/ethtool/ethtool_linux.go @@ -82,12 +82,38 @@ func (e *Ethtool) gatherEthtoolStats(iface net.Interface, acc telegraf.Accumulat fields[fieldInterfaceUp] = e.interfaceUp(iface) for k, v := range stats { - fields[strings.TrimSpace(k)] = v + fields[e.normalizeKey(k)] = v } acc.AddFields(pluginName, fields, tags) } +// normalize key string; order matters to avoid replacing whitespace with +// underscores, then trying to trim those same underscores. +func (e *Ethtool) normalizeKey(key string) string { + if inStringSlice(e.NormalizeKeys, "trim") { + key = strings.TrimSpace(key) + } + if inStringSlice(e.NormalizeKeys, "lower") { + key = strings.ToLower(key) + } + if inStringSlice(e.NormalizeKeys, "spaces") { + key = strings.ReplaceAll(key, " ", "_") + } + + return key +} + +func inStringSlice(slice []string, value string) bool { + for _, item := range slice { + if item == value { + return true + } + } + + return false +} + func (e *Ethtool) interfaceUp(iface net.Interface) bool { return (iface.Flags & net.FlagUp) != 0 } diff --git a/plugins/inputs/ethtool/ethtool_test.go b/plugins/inputs/ethtool/ethtool_test.go index 14cf14d811683..3c04412d83f37 100644 --- a/plugins/inputs/ethtool/ethtool_test.go +++ b/plugins/inputs/ethtool/ethtool_test.go @@ -380,3 +380,94 @@ func TestGatherIgnoreInterfaces(t *testing.T) { } acc.AssertContainsTaggedFields(t, pluginName, expectedFieldsEth2, expectedTagsEth2) } + +type TestCase struct { + normalization []string + stats map[string]uint64 + expectedFields map[string]uint64 +} + +func TestNormalizedKeys(t *testing.T) { + cases := []TestCase{ + { + normalization: []string{"spaces"}, + stats: map[string]uint64{ + "port rx": 1, + " Port_tx": 0, + "interface_up": 0, + }, + expectedFields: map[string]uint64{ + "port_rx": 1, + "_Port_tx": 0, + "interface_up": 0, + }, + }, + { + normalization: []string{"spaces", "lower"}, + stats: map[string]uint64{ + "Port rx": 1, + " Port_tx": 0, + "interface_up": 0, + }, + expectedFields: map[string]uint64{ + "port_rx": 1, + "_port_tx": 0, + "interface_up": 0, + }, + }, + { + normalization: []string{"spaces", "lower", "trim"}, + stats: map[string]uint64{ + " Port RX ": 1, + " Port_tx": 0, + "interface_up": 0, + }, + expectedFields: map[string]uint64{ + "port_rx": 1, + "port_tx": 0, + "interface_up": 0, + }, + }, + { + normalization: []string{}, + stats: map[string]uint64{ + " Port RX ": 1, + " Port_tx": 0, + "interface_up": 0, + }, + expectedFields: map[string]uint64{ + " Port RX ": 1, + " Port_tx": 0, + "interface_up": 0, + }, + }, + } + + for _, c := range cases { + eth0 := &InterfaceMock{"eth0", "e1000e", c.stats, false, true} + expectedTags := map[string]string{ + "interface": eth0.Name, + "driver": eth0.DriverName, + } + + interfaceMap = make(map[string]*InterfaceMock) + interfaceMap[eth0.Name] = eth0 + + cmd := &CommandEthtoolMock{interfaceMap} + command = &Ethtool{ + InterfaceInclude: []string{}, + InterfaceExclude: []string{}, + NormalizeKeys: c.normalization, + command: cmd, + } + + var acc testutil.Accumulator + err := command.Gather(&acc) + + assert.NoError(t, err) + assert.Len(t, acc.Metrics, 1) + + acc.AssertContainsTaggedFields(t, pluginName, toStringMapInterface(c.expectedFields), expectedTags) + } + +} From a381c6831a4830b42bfcc5bca4b29e0cbd936cdc Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Tue, 12 Oct 2021 16:30:31 -0600 Subject: [PATCH 3/5] remote whitespace --- plugins/inputs/ethtool/ethtool_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/inputs/ethtool/ethtool_test.go b/plugins/inputs/ethtool/ethtool_test.go index 3c04412d83f37..b5d9433a68e4a 100644 --- a/plugins/inputs/ethtool/ethtool_test.go +++ b/plugins/inputs/ethtool/ethtool_test.go @@ -442,7 +442,6 @@ func TestNormalizedKeys(t *testing.T) { }, }, } - for _, c := range cases { eth0 := &InterfaceMock{"eth0", "e1000e", c.stats, false, true} expectedTags := map[string]string{ @@ -469,5 +468,4 @@ func TestNormalizedKeys(t *testing.T) { acc.AssertContainsTaggedFields(t, pluginName, toStringMapInterface(c.expectedFields), expectedTags) } - } From ea7817c5f78de0942be0b1f478b95ab0b78fd38b Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Fri, 15 Oct 2021 13:02:05 -0600 Subject: [PATCH 4/5] rename spaces to underscore, add snakecase option --- plugins/inputs/ethtool/README.md | 5 ++-- plugins/inputs/ethtool/ethtool.go | 5 ++-- plugins/inputs/ethtool/ethtool_linux.go | 20 +++++++++++++-- plugins/inputs/ethtool/ethtool_test.go | 33 ++++++++++++++++++++++--- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/plugins/inputs/ethtool/README.md b/plugins/inputs/ethtool/README.md index 6d5b36d1b5e31..218b1323530e9 100644 --- a/plugins/inputs/ethtool/README.md +++ b/plugins/inputs/ethtool/README.md @@ -16,10 +16,11 @@ The ethtool input plugin pulls ethernet device stats. Fields pulled will depend ## Some drivers declare statistics with extra whitespace, different spacing, ## and mix cases. This list, when enabled, can be used to clean the keys. ## Here are the current possible normalizations: + ## * snakecase: converts camelCaseWords to camel_case_words ## * trim: removes leading and trailing whitespace ## * lower: changes all capitalized letters to lowercase - ## * spaces: replaces spaces with underscores - # normalize_keys = ["trim", "lower", "spaces"] + ## * underscore: replaces spaces with underscores + # normalize_keys = ["snakecase", "trim", "lower", "underscore"] ``` Interfaces can be included or ignored using: diff --git a/plugins/inputs/ethtool/ethtool.go b/plugins/inputs/ethtool/ethtool.go index fa54f1e472bf0..4f9f5d28b6e47 100644 --- a/plugins/inputs/ethtool/ethtool.go +++ b/plugins/inputs/ethtool/ethtool.go @@ -45,10 +45,11 @@ const ( ## Some drivers declare statistics with extra whitespace, different spacing, ## and mix cases. This list, when enabled, can be used to clean the keys. ## Here are the current possible normalizations: + ## * snakecase: converts camelCaseWords to camel_case_words ## * trim: removes leading and trailing whitespace ## * lower: changes all capitalized letters to lowercase - ## * spaces: replaces spaces with underscores - # normalize_keys = ["trim", "lower", "spaces"] + ## * underscore: replaces spaces with underscores + # normalize_keys = ["snakecase", "trim", "lower", "underscore"] ` ) diff --git a/plugins/inputs/ethtool/ethtool_linux.go b/plugins/inputs/ethtool/ethtool_linux.go index a3814043eff9c..16081e4cd831a 100644 --- a/plugins/inputs/ethtool/ethtool_linux.go +++ b/plugins/inputs/ethtool/ethtool_linux.go @@ -5,6 +5,7 @@ package ethtool import ( "net" + "regexp" "strings" "sync" @@ -89,21 +90,36 @@ func (e *Ethtool) gatherEthtoolStats(iface net.Interface, acc telegraf.Accumulat } // normalize key string; order matters to avoid replacing whitespace with -// underscores, then trying to trim those same underscores. +// underscores, then trying to trim those same underscores. Likewise with +// camelcase before trying to lower case things. func (e *Ethtool) normalizeKey(key string) string { + // must trim whitespace or this will have a leading _ + if inStringSlice(e.NormalizeKeys, "snakecase") { + key = camelCase2SnakeCase(strings.TrimSpace(key)) + } + // must occur before underscore, otherwise nothing to trim if inStringSlice(e.NormalizeKeys, "trim") { key = strings.TrimSpace(key) } if inStringSlice(e.NormalizeKeys, "lower") { key = strings.ToLower(key) } - if inStringSlice(e.NormalizeKeys, "spaces") { + if inStringSlice(e.NormalizeKeys, "underscore") { key = strings.ReplaceAll(key, " ", "_") } return key } +func camelCase2SnakeCase(value string) string { + matchFirstCap := regexp.MustCompile("(.)([A-Z][a-z]+)") + matchAllCap := regexp.MustCompile("([a-z0-9])([A-Z])") + + snake := matchFirstCap.ReplaceAllString(value, "${1}_${2}") + snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}") + return strings.ToLower(snake) +} + func inStringSlice(slice []string, value string) bool { for _, item := range slice { if item == value { diff --git a/plugins/inputs/ethtool/ethtool_test.go b/plugins/inputs/ethtool/ethtool_test.go index b5d9433a68e4a..f9573ee054429 100644 --- a/plugins/inputs/ethtool/ethtool_test.go +++ b/plugins/inputs/ethtool/ethtool_test.go @@ -390,7 +390,7 @@ type TestCase struct { func TestNormalizedKeys(t *testing.T) { cases := []TestCase{ { - normalization: []string{"spaces"}, + normalization: []string{"underscore"}, stats: map[string]uint64{ "port rx": 1, " Port_tx": 0, @@ -403,7 +403,7 @@ func TestNormalizedKeys(t *testing.T) { }, }, { - normalization: []string{"spaces", "lower"}, + normalization: []string{"underscore", "lower"}, stats: map[string]uint64{ "Port rx": 1, " Port_tx": 0, @@ -416,7 +416,7 @@ func TestNormalizedKeys(t *testing.T) { }, }, { - normalization: []string{"spaces", "lower", "trim"}, + normalization: []string{"underscore", "lower", "trim"}, stats: map[string]uint64{ " Port RX ": 1, " Port_tx": 0, @@ -428,6 +428,32 @@ func TestNormalizedKeys(t *testing.T) { "interface_up": 0, }, }, + { + normalization: []string{"underscore", "lower", "snakecase", "trim"}, + stats: map[string]uint64{ + " Port RX ": 1, + " Port_tx": 0, + "interface_up": 0, + }, + expectedFields: map[string]uint64{ + "port_rx": 1, + "port_tx": 0, + "interface_up": 0, + }, + }, + { + normalization: []string{"snakecase"}, + stats: map[string]uint64{ + " PortRX ": 1, + " PortTX": 0, + "interface_up": 0, + }, + expectedFields: map[string]uint64{ + "port_rx": 1, + "port_tx": 0, + "interface_up": 0, + }, + }, { normalization: []string{}, stats: map[string]uint64{ @@ -466,6 +492,7 @@ func TestNormalizedKeys(t *testing.T) { assert.NoError(t, err) assert.Len(t, acc.Metrics, 1) + acc.AssertContainsFields(t, pluginName, toStringMapInterface(c.expectedFields)) acc.AssertContainsTaggedFields(t, pluginName, toStringMapInterface(c.expectedFields), expectedTags) } } From 82795fba583a6ab5d4da2532013374803ab35f9d Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Mon, 18 Oct 2021 12:51:55 -0600 Subject: [PATCH 5/5] update doc example to be more clear --- plugins/inputs/ethtool/README.md | 2 +- plugins/inputs/ethtool/ethtool.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/ethtool/README.md b/plugins/inputs/ethtool/README.md index 218b1323530e9..333630c958703 100644 --- a/plugins/inputs/ethtool/README.md +++ b/plugins/inputs/ethtool/README.md @@ -16,7 +16,7 @@ The ethtool input plugin pulls ethernet device stats. Fields pulled will depend ## Some drivers declare statistics with extra whitespace, different spacing, ## and mix cases. This list, when enabled, can be used to clean the keys. ## Here are the current possible normalizations: - ## * snakecase: converts camelCaseWords to camel_case_words + ## * snakecase: converts fooBarBaz to foo_bar_baz ## * trim: removes leading and trailing whitespace ## * lower: changes all capitalized letters to lowercase ## * underscore: replaces spaces with underscores diff --git a/plugins/inputs/ethtool/ethtool.go b/plugins/inputs/ethtool/ethtool.go index 4f9f5d28b6e47..256652640f383 100644 --- a/plugins/inputs/ethtool/ethtool.go +++ b/plugins/inputs/ethtool/ethtool.go @@ -45,7 +45,7 @@ const ( ## Some drivers declare statistics with extra whitespace, different spacing, ## and mix cases. This list, when enabled, can be used to clean the keys. ## Here are the current possible normalizations: - ## * snakecase: converts camelCaseWords to camel_case_words + ## * snakecase: converts fooBarBaz to foo_bar_baz ## * trim: removes leading and trailing whitespace ## * lower: changes all capitalized letters to lowercase ## * underscore: replaces spaces with underscores