Get stats for all indices, including hidden indices (#18703)

* Get stats for all indices

* Update default stack versions to use in integration tests

* Add integration test for hidden indices stats

* Better naming of variables

* Adding CHANGELOG entry

* Use expand_wildcards in tests as well

* Collect stats for open and hidden indices (but not closed ones)

CHANGELOG.next.asciidoc

@@ -527,6 +527,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboards for googlecloud load balancing metricset. {pull}18369[18369]
 - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268]
 - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423]
+- The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703]
 
 *Packetbeat*
 

metricbeat/docker-compose.yml

@@ -15,11 +15,11 @@ services:
 
   # Used by base tests
   elasticsearch:
-    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.5.2}-1
+    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.7.0}-1
     build:
       context: ./module/elasticsearch/_meta
       args:
-        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.5.2}
+        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.7.0}
     environment:
       - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
       - "network.host="
@@ -37,23 +37,23 @@ services:
 
   # Used by base tests
   kibana:
-    image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.5.2}-1
+    image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.7.0}-1
     build:
       context: ./module/kibana/_meta
       args:
-        KIBANA_VERSION: ${KIBANA_VERSION:-7.5.2}
+        KIBANA_VERSION: ${KIBANA_VERSION:-7.7.0}
     depends_on:
       - elasticsearch
     ports:
       - 5601
 
   # Used by base tests
   metricbeat:
-    image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-7.5.2}-1
+    image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-7.7.0}-1
     build:
       context: ./module/beat/_meta
       args:
-        BEAT_VERSION: ${BEAT_VERSION:-7.5.2}
+        BEAT_VERSION: ${BEAT_VERSION:-7.7.0}
     command: '-e'
     ports:
       - 5066

metricbeat/module/elasticsearch/elasticsearch_integration_test.go

@@ -24,7 +24,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
@@ -40,7 +42,7 @@ import (
 	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/ccr"
 	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/cluster_stats"
 	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/enrich"
-	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/index"
+	"github.com/elastic/beats/v7/metricbeat/module/elasticsearch/index"
 	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/index_recovery"
 	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/index_summary"
 	_ "github.com/elastic/beats/v7/metricbeat/module/elasticsearch/ml_job"
@@ -185,6 +187,55 @@ func TestXPackEnabled(t *testing.T) {
 	}
 }
 
+func TestGetAllIndices(t *testing.T) {
+	service := compose.EnsureUpWithTimeout(t, 300, "elasticsearch")
+	host := service.Host()
+
+	// Create two indices, one hidden, one not
+	indexVisible, err := createIndex(host, false)
+	require.NoError(t, err)
+
+	indexHidden, err := createIndex(host, true)
+	require.NoError(t, err)
+
+	config := getXPackConfig(host)
+
+	metricSets := mbtest.NewReportingMetricSetV2Errors(t, config)
+	for _, metricSet := range metricSets {
+		// We only care about the index metricset for this test
+		if metricSet.Name() != "index" {
+			continue
+		}
+
+		events, errs := mbtest.ReportingFetchV2Error(metricSet)
+
+		require.Empty(t, errs)
+		require.NotEmpty(t, events)
+
+		// Check that we have events for both indices we created
+		var idxVisibleExists, idxHiddenExists bool
+		for _, event := range events {
+			v, err := event.RootFields.GetValue("index_stats")
+			require.NoError(t, err)
+
+			idx, ok := v.(index.Index)
+			if !ok {
+				t.FailNow()
+			}
+
+			switch idx.Index {
+			case indexVisible:
+				idxVisibleExists = true
+			case indexHidden:
+				idxHiddenExists = true
+			}
+		}
+
+		require.True(t, idxVisibleExists)
+		require.True(t, idxHiddenExists)
+	}
+}
+
 // GetConfig returns config for elasticsearch module
 func getConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
@@ -205,7 +256,7 @@ func getXPackConfig(host string) map[string]interface{} {
 }
 
 func setupTest(t *testing.T, esHost string, esVersion *common.Version) {
-	err := createIndex(esHost)
+	_, err := createIndex(esHost, false)
 	require.NoError(t, err)
 
 	err = enableTrialLicense(esHost, esVersion)
@@ -221,30 +272,31 @@ func setupTest(t *testing.T, esHost string, esVersion *common.Version) {
 	require.NoError(t, err)
 }
 
-// createIndex creates and elasticsearch index in case it does not exit yet
-func createIndex(host string) error {
-	client := &http.Client{}
+// createIndex creates an random elasticsearch index
+func createIndex(host string, isHidden bool) (string, error) {
+	indexName := randString(5)
 
-	if checkExists("http://" + host + "/testindex") {
-		return nil
-	}
+	reqBody := fmt.Sprintf(`{ "settings": { "index.hidden": %v } }`, isHidden)
 
-	req, err := http.NewRequest("PUT", "http://"+host+"/testindex", nil)
+	req, err := http.NewRequest("PUT", fmt.Sprintf("http://%v/%v", host, indexName), strings.NewReader(reqBody))
 	if err != nil {
-		return err
+		return "", errors.Wrap(err, "could not build create index request")
 	}
+	req.Header.Add("Content-Type", "application/json")
 
+	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		return err
+		return "", errors.Wrap(err, "could not send create index request")
 	}
 	defer resp.Body.Close()
+	respBody, err := ioutil.ReadAll(resp.Body)
 
 	if resp.StatusCode != 200 {
-		return fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status)
+		return "", fmt.Errorf("HTTP error %d: %s, %s", resp.StatusCode, resp.Status, string(respBody))
 	}
 
-	return nil
+	return indexName, nil
 }
 
 // enableTrialLicense creates and elasticsearch index in case it does not exit yet
@@ -539,16 +591,16 @@ func ingestAndEnrichDoc(host string) error {
 }
 
 func countIndices(elasticsearchHostPort string) (int, error) {
-	return countCatItems(elasticsearchHostPort, "indices")
+	return countCatItems(elasticsearchHostPort, "indices", "&expand_wildcards=open,hidden")
 
 }
 
 func countShards(elasticsearchHostPort string) (int, error) {
-	return countCatItems(elasticsearchHostPort, "shards")
+	return countCatItems(elasticsearchHostPort, "shards", "")
 }
 
-func countCatItems(elasticsearchHostPort, catObject string) (int, error) {
-	resp, err := http.Get("http://" + elasticsearchHostPort + "/_cat/" + catObject + "?format=json")
+func countCatItems(elasticsearchHostPort, catObject, extraParams string) (int, error) {
+	resp, err := http.Get("http://" + elasticsearchHostPort + "/_cat/" + catObject + "?format=json" + extraParams)
 	if err != nil {
 		return 0, err
 	}
@@ -659,3 +711,16 @@ func waitForSuccess(f checkSuccessFunction, retryInterval time.Duration, numAtte
 
 	return false, nil
 }
+
+func randString(len int) string {
+	rand.Seed(time.Now().UnixNano())
+
+	b := make([]byte, len)
+	aIdx := int('a')
+	for i := range b {
+		charIdx := aIdx + rand.Intn(26)
+		b[i] = byte(charIdx)
+	}
+
+	return string(b)
+}

metricbeat/module/elasticsearch/index/data_xpack.go

@@ -38,10 +38,10 @@ var (
 
 // Based on https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L127-L203
 type stats struct {
-	Indices map[string]index `json:"indices"`
+	Indices map[string]Index `json:"indices"`
 }
 
-type index struct {
+type Index struct {
 	UUID      string     `json:"uuid"`
 	Primaries indexStats `json:"primaries"`
 	Total     indexStats `json:"total"`
@@ -173,7 +173,7 @@ func parseAPIResponse(content []byte, indicesStats *stats) error {
 
 // Fields added here are based on same fields being added by internal collection in
 // https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L62-L124
-func addClusterStateFields(idx *index, clusterState common.MapStr) error {
+func addClusterStateFields(idx *Index, clusterState common.MapStr) error {
 	indexMetadata, err := getClusterStateMetricForIndex(clusterState, idx.Index, "metadata")
 	if err != nil {
 		return errors.Wrap(err, "failed to get index metadata from cluster state")

metricbeat/module/elasticsearch/index/index.go

@@ -39,7 +39,7 @@ func init() {
 
 const (
 	statsMetrics = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache"
-	statsPath    = "/_stats/" + statsMetrics + "?filter_path=indices"
+	statsPath    = "/_stats/" + statsMetrics + "?filter_path=indices&expand_wildcards=open,hidden"
 )
 
 // MetricSet type defines all fields of the MetricSet

metricbeat/module/logstash/docker-compose.yml

@@ -2,22 +2,22 @@ version: '2.3'
 
 services:
   logstash:
-    image: docker.elastic.co/integrations-ci/beats-logstash:${LOGSTASH_VERSION:-7.5.2}-1
+    image: docker.elastic.co/integrations-ci/beats-logstash:${LOGSTASH_VERSION:-7.7.0}-1
     build:
       context: ./_meta
       args:
-        LOGSTASH_VERSION: ${LOGSTASH_VERSION:-7.5.2}
+        LOGSTASH_VERSION: ${LOGSTASH_VERSION:-7.7.0}
     ports:
       - 9600
     depends_on:
       - elasticsearch
 
   elasticsearch:
-    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.5.2}-1
+    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.7.0}-1
     build:
       context: ../elasticsearch/_meta
       args:
-        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.5.2}
+        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.7.0}
     environment:
       - "network.host="
       - "transport.host=127.0.0.1"

x-pack/metricbeat/docker-compose.yml

@@ -24,11 +24,11 @@ services:
   kibana:
     # Copied configuration from OSS metricbeat because services with depends_on
     # cannot be extended with extends
-    image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.5.2}-1
+    image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.7.0}-1
     build:
       context: ../../metricbeat/module/kibana/_meta
       args:
-        KIBANA_VERSION: ${KIBANA_VERSION:-7.5.2}
+        KIBANA_VERSION: ${KIBANA_VERSION:-7.7.0}
     depends_on:
       - elasticsearch
     ports: