Report if index is hidden in index stats (elastic#18706)

* Report index hidden setting in index stats

* Adding CHANGELOG entry

* Use expand_wildcards in tests as well

* Using json.Unmarshaler instead

* Hybrid approach

CHANGELOG.next.asciidoc

@@ -539,6 +539,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423]
 - Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660]
 - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703]
+- The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706]
 
 *Packetbeat*
 

metricbeat/module/elasticsearch/elasticsearch.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -359,6 +360,61 @@ func GetXPack(http *helper.HTTP, resetURI string) (XPack, error) {
 	return xpack, err
 }
 
+type boolStr bool
+
+func (b *boolStr) UnmarshalJSON(raw []byte) error {
+	var bs string
+	err := json.Unmarshal(raw, &bs)
+	if err != nil {
+		return err
+	}
+
+	bv, err := strconv.ParseBool(bs)
+	if err != nil {
+		return err
+	}
+
+	*b = boolStr(bv)
+	return nil
+}
+
+type IndexSettings struct {
+	Hidden bool
+}
+
+// GetIndicesSettings returns a map of index names to their settings.
+// Note that as of now it is optimized to fetch only the "hidden" index setting to keep the memory
+// footprint of this function call as low as possible.
+func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSettings, error) {
+	content, err := fetchPath(http, resetURI, "*/_settings", "filter_path=*.settings.index.hidden&expand_wildcards=all")
+
+	if err != nil {
+		return nil, errors.Wrap(err, "could not fetch indices settings")
+	}
+
+	var resp map[string]struct {
+		Settings struct {
+			Index struct {
+				Hidden boolStr `json:"hidden"`
+			} `json:"index"`
+		} `json:"settings"`
+	}
+
+	err = json.Unmarshal(content, &resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse indices settings response")
+	}
+
+	ret := make(map[string]IndexSettings, len(resp))
+	for index, settings := range resp {
+		ret[index] = IndexSettings{
+			Hidden: bool(settings.Settings.Index.Hidden),
+		}
+	}
+
+	return ret, nil
+}
+
 // IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled
 func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) {
 	content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall")

metricbeat/module/elasticsearch/elasticsearch_integration_test.go

@@ -226,8 +226,10 @@ func TestGetAllIndices(t *testing.T) {
 			switch idx.Index {
 			case indexVisible:
 				idxVisibleExists = true
+				require.False(t, idx.Hidden)
 			case indexHidden:
 				idxHiddenExists = true
+				require.True(t, idx.Hidden)
 			}
 		}
 
@@ -592,7 +594,6 @@ func ingestAndEnrichDoc(host string) error {
 
 func countIndices(elasticsearchHostPort string) (int, error) {
 	return countCatItems(elasticsearchHostPort, "indices", "&expand_wildcards=open,hidden")
-
 }
 
 func countShards(elasticsearchHostPort string) (int, error) {

metricbeat/module/elasticsearch/index/data_xpack.go

@@ -49,6 +49,7 @@ type Index struct {
 	Index   string     `json:"index"`
 	Created int64      `json:"created"`
 	Status  string     `json:"status"`
+	Hidden  bool       `json:"hidden"`
 	Shards  shardStats `json:"shards"`
 }
 
@@ -141,11 +142,21 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info,
 		return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response")
 	}
 
+	indicesSettings, err := elasticsearch.GetIndicesSettings(m.HTTP, m.HTTP.GetURI())
+	if err != nil {
+		return errors.Wrap(err, "failure retrieving indices settings from Elasticsearch")
+	}
+
 	var errs multierror.Errors
 	for name, idx := range indicesStats.Indices {
 		event := mb.Event{}
 		idx.Index = name
 
+		settings, exists := indicesSettings[name]
+		if exists {
+			idx.Hidden = settings.Hidden
+		}
+
 		err = addClusterStateFields(&idx, clusterState)
 		if err != nil {
 			errs = append(errs, errors.Wrap(err, "failure adding cluster state fields"))