diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5cab9da8759..5aaeb53ba2c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -534,6 +534,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for named ports in autodiscover. {pull}19398[19398] - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] +- The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706] *Packetbeat* diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 07057bb147f..48050b224cc 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net/url" + "strconv" "strings" "sync" "time" @@ -359,6 +360,61 @@ func GetXPack(http *helper.HTTP, resetURI string) (XPack, error) { return xpack, err } +type boolStr bool + +func (b *boolStr) UnmarshalJSON(raw []byte) error { + var bs string + err := json.Unmarshal(raw, &bs) + if err != nil { + return err + } + + bv, err := strconv.ParseBool(bs) + if err != nil { + return err + } + + *b = boolStr(bv) + return nil +} + +type IndexSettings struct { + Hidden bool +} + +// GetIndicesSettings returns a map of index names to their settings. +// Note that as of now it is optimized to fetch only the "hidden" index setting to keep the memory +// footprint of this function call as low as possible. +func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSettings, error) { + content, err := fetchPath(http, resetURI, "*/_settings", "filter_path=*.settings.index.hidden&expand_wildcards=all") + + if err != nil { + return nil, errors.Wrap(err, "could not fetch indices settings") + } + + var resp map[string]struct { + Settings struct { + Index struct { + Hidden boolStr `json:"hidden"` + } `json:"index"` + } `json:"settings"` + } + + err = json.Unmarshal(content, &resp) + if err != nil { + return nil, errors.Wrap(err, "could not parse indices settings response") + } + + ret := make(map[string]IndexSettings, len(resp)) + for index, settings := range resp { + ret[index] = IndexSettings{ + Hidden: bool(settings.Settings.Index.Hidden), + } + } + + return ret, nil +} + // IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) { content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall") diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index 982fdebeff8..57a0e294773 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -226,8 +226,10 @@ func TestGetAllIndices(t *testing.T) { switch idx.Index { case indexVisible: idxVisibleExists = true + require.False(t, idx.Hidden) case indexHidden: idxHiddenExists = true + require.True(t, idx.Hidden) } } @@ -592,7 +594,6 @@ func ingestAndEnrichDoc(host string) error { func countIndices(elasticsearchHostPort string) (int, error) { return countCatItems(elasticsearchHostPort, "indices", "&expand_wildcards=open,hidden") - } func countShards(elasticsearchHostPort string) (int, error) { diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index 76d6f11002c..b137e06364b 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -49,6 +49,7 @@ type Index struct { Index string `json:"index"` Created int64 `json:"created"` Status string `json:"status"` + Hidden bool `json:"hidden"` Shards shardStats `json:"shards"` } @@ -141,11 +142,21 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response") } + indicesSettings, err := elasticsearch.GetIndicesSettings(m.HTTP, m.HTTP.GetURI()) + if err != nil { + return errors.Wrap(err, "failure retrieving indices settings from Elasticsearch") + } + var errs multierror.Errors for name, idx := range indicesStats.Indices { event := mb.Event{} idx.Index = name + settings, exists := indicesSettings[name] + if exists { + idx.Hidden = settings.Hidden + } + err = addClusterStateFields(&idx, clusterState) if err != nil { errs = append(errs, errors.Wrap(err, "failure adding cluster state fields"))