From abc108b2cba8749cd97422096c99d17f113aef47 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 22 May 2020 09:52:04 -0400 Subject: [PATCH 1/5] Use expand_wildcards in tests as well --- .../module/elasticsearch/elasticsearch_integration_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index 982fdebeff8..96775e9b183 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -592,7 +592,6 @@ func ingestAndEnrichDoc(host string) error { func countIndices(elasticsearchHostPort string) (int, error) { return countCatItems(elasticsearchHostPort, "indices", "&expand_wildcards=open,hidden") - } func countShards(elasticsearchHostPort string) (int, error) { From c8c35adfcb6f4976ed118519c745bcf5e1a00c3e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 22 May 2020 09:20:28 -0400 Subject: [PATCH 2/5] Report index hidden setting in index stats --- .../module/elasticsearch/elasticsearch.go | 40 +++++++++++++++++++ .../elasticsearch_integration_test.go | 2 + .../module/elasticsearch/index/data_xpack.go | 11 +++++ 3 files changed, 53 insertions(+) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 07057bb147f..423f45a0af6 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net/url" + "strconv" "strings" "sync" "time" @@ -359,6 +360,45 @@ func GetXPack(http *helper.HTTP, resetURI string) (XPack, error) { return xpack, err } +type IndexSettings struct { + Hidden bool +} + +// GetIndicesSettings returns a map of index names to their settings. +// Note that as of now it is optimized to fetch only the "hidden" index setting to keep the memory +// footprint of this function call as low as possible. +func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSettings, error) { + content, err := fetchPath(http, resetURI, "*/_settings", "filter_path=*.settings.index.hidden&expand_wildcards=all") + + if err != nil { + return nil, errors.Wrap(err, "could not fetch indices settings") + } + + var resp map[string]struct { + Settings struct { + Index struct { + Hidden string `json:"hidden"` + } `json:"index"` + } `json:"settings"` + } + + err = json.Unmarshal(content, &resp) + if err != nil { + return nil, errors.Wrap(err, "could not parse indices settings response") + } + + ret := make(map[string]IndexSettings, len(resp)) + for index, settings := range resp { + isHidden, err := strconv.ParseBool(settings.Settings.Index.Hidden) + if err != nil { + return nil, errors.Wrap(err, "could not parse hidden setting as boolean") + } + ret[index] = IndexSettings{Hidden: isHidden} + } + + return ret, nil +} + // IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) { content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall") diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index 96775e9b183..57a0e294773 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -226,8 +226,10 @@ func TestGetAllIndices(t *testing.T) { switch idx.Index { case indexVisible: idxVisibleExists = true + require.False(t, idx.Hidden) case indexHidden: idxHiddenExists = true + require.True(t, idx.Hidden) } } diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index 76d6f11002c..b137e06364b 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -49,6 +49,7 @@ type Index struct { Index string `json:"index"` Created int64 `json:"created"` Status string `json:"status"` + Hidden bool `json:"hidden"` Shards shardStats `json:"shards"` } @@ -141,11 +142,21 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response") } + indicesSettings, err := elasticsearch.GetIndicesSettings(m.HTTP, m.HTTP.GetURI()) + if err != nil { + return errors.Wrap(err, "failure retrieving indices settings from Elasticsearch") + } + var errs multierror.Errors for name, idx := range indicesStats.Indices { event := mb.Event{} idx.Index = name + settings, exists := indicesSettings[name] + if exists { + idx.Hidden = settings.Hidden + } + err = addClusterStateFields(&idx, clusterState) if err != nil { errs = append(errs, errors.Wrap(err, "failure adding cluster state fields")) From 987c58366633e0ec7b726c7c685d7e401415ba8e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 22 May 2020 09:27:13 -0400 Subject: [PATCH 3/5] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5cab9da8759..5aaeb53ba2c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -534,6 +534,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for named ports in autodiscover. {pull}19398[19398] - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] +- The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706] *Packetbeat* From 87b73dda85a7f0d5d1064219bfa7edcd57d8b580 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 7 Jul 2020 06:56:04 -0700 Subject: [PATCH 4/5] Using json.Unmarshaler instead --- .../module/elasticsearch/elasticsearch.go | 30 +++++++++++++------ .../module/elasticsearch/index/data_xpack.go | 2 +- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 423f45a0af6..10bcc13ca61 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -360,8 +360,26 @@ func GetXPack(http *helper.HTTP, resetURI string) (XPack, error) { return xpack, err } +type boolStr bool + +func (b *boolStr) UnmarshalJSON(raw []byte) error { + var bs string + err := json.Unmarshal(raw, &bs) + if err != nil { + return err + } + + bv, err := strconv.ParseBool(bs) + if err != nil { + return err + } + + *b = boolStr(bv) + return nil +} + type IndexSettings struct { - Hidden bool + Hidden boolStr } // GetIndicesSettings returns a map of index names to their settings. @@ -376,9 +394,7 @@ func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSet var resp map[string]struct { Settings struct { - Index struct { - Hidden string `json:"hidden"` - } `json:"index"` + Index IndexSettings `json:"index"` } `json:"settings"` } @@ -389,11 +405,7 @@ func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSet ret := make(map[string]IndexSettings, len(resp)) for index, settings := range resp { - isHidden, err := strconv.ParseBool(settings.Settings.Index.Hidden) - if err != nil { - return nil, errors.Wrap(err, "could not parse hidden setting as boolean") - } - ret[index] = IndexSettings{Hidden: isHidden} + ret[index] = settings.Settings.Index } return ret, nil diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index b137e06364b..46e41764d9a 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -154,7 +154,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, settings, exists := indicesSettings[name] if exists { - idx.Hidden = settings.Hidden + idx.Hidden = bool(settings.Hidden) } err = addClusterStateFields(&idx, clusterState) From eb621b5ce825a622aae008506d1ee3507be44c94 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 7 Jul 2020 08:42:57 -0700 Subject: [PATCH 5/5] Hybrid approach --- metricbeat/module/elasticsearch/elasticsearch.go | 10 +++++++--- metricbeat/module/elasticsearch/index/data_xpack.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 10bcc13ca61..48050b224cc 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -379,7 +379,7 @@ func (b *boolStr) UnmarshalJSON(raw []byte) error { } type IndexSettings struct { - Hidden boolStr + Hidden bool } // GetIndicesSettings returns a map of index names to their settings. @@ -394,7 +394,9 @@ func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSet var resp map[string]struct { Settings struct { - Index IndexSettings `json:"index"` + Index struct { + Hidden boolStr `json:"hidden"` + } `json:"index"` } `json:"settings"` } @@ -405,7 +407,9 @@ func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSet ret := make(map[string]IndexSettings, len(resp)) for index, settings := range resp { - ret[index] = settings.Settings.Index + ret[index] = IndexSettings{ + Hidden: bool(settings.Settings.Index.Hidden), + } } return ret, nil diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index 46e41764d9a..b137e06364b 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -154,7 +154,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, settings, exists := indicesSettings[name] if exists { - idx.Hidden = bool(settings.Hidden) + idx.Hidden = settings.Hidden } err = addClusterStateFields(&idx, clusterState)