[AWS Cloudwatch] Changed module to call ListMetrics API only once per region

CHANGELOG.next.asciidoc

@@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Add support for multiple regions in GCP {pull}32964[32964]
 - Add GCP Redis regions support {pull}33728[33728]
 - Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763]
+- Changed cloudwatch module to call ListMetrics API only once per region, instead of per AWS namespace {pull}34055[34055]
 
 *Packetbeat*
 

metricbeat/docs/modules/aws.asciidoc

@@ -297,7 +297,7 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 | IAM ListAccountAliases | 1 | Once on startup
 | STS GetCallerIdentity | 1 | Once on startup
 | EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per namespace per collection period
+| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
 | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -285,7 +285,7 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 | IAM ListAccountAliases | 1 | Once on startup
 | STS GetCallerIdentity | 1 | Once on startup
 | EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per namespace per collection period
+| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
 | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -180,10 +180,13 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 		if err != nil {
 			m.Logger().Warn("skipping metrics list from region '%s'", regionName)
 		}
+
+		// retrieve all the details for all the metrics available in the current region
+		listMetricsOutput, _ := aws.GetListMetricsOutput("*", regionName, m.Period, svcCloudwatch)
+
 		for namespace, namespaceDetails := range namespaceDetailTotal {
 			m.logger.Debugf("Collected metrics from namespace %s", namespace)
 
-			listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, svcCloudwatch)
 			if err != nil {
 				m.logger.Info(err.Error())
 				continue
@@ -192,9 +195,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 			if len(listMetricsOutput) == 0 {
 				continue
 			}
-
 			// filter listMetricsOutput by detailed configuration per each namespace
-			filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespaceDetails)
+			filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespace, namespaceDetails)
+
 			// get resource type filters and tags filters for each namespace
 			resourceTypeTagFilters := constructTagsFilters(namespaceDetails)
 
@@ -240,52 +243,28 @@ func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionNa
 }
 
 // filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match
-func filterListMetricsOutput(listMetricsOutput []types.Metric, namespaceDetails []namespaceDetail) []metricsWithStatistics {
+func filterListMetricsOutput(listMetricsOutput []types.Metric, namespace string, namespaceDetails []namespaceDetail) []metricsWithStatistics {
 	var filteredMetricWithStatsTotal []metricsWithStatistics
 	for _, listMetric := range listMetricsOutput {
-		for _, configPerNamespace := range namespaceDetails {
-			if configPerNamespace.names != nil && configPerNamespace.dimensions == nil {
-				// if metric names are given in config but no dimensions, filter
-				// out the metrics with other names
-				if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
-					continue
-				}
-				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
-					metricsWithStatistics{
-						cloudwatchMetric: listMetric,
-						statistic:        configPerNamespace.statistics,
-					})
-
-			} else if configPerNamespace.names == nil && configPerNamespace.dimensions != nil {
-				// if metric names are not given in config but dimensions are
-				// given, only keep the metrics with matching dimensions
-				if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
-					continue
-				}
-				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
-					metricsWithStatistics{
-						cloudwatchMetric: listMetric,
-						statistic:        configPerNamespace.statistics,
-					})
-			} else if configPerNamespace.names != nil && configPerNamespace.dimensions != nil {
-				if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
-					continue
+		if *listMetric.Namespace == namespace {
+			for _, configPerNamespace := range namespaceDetails {
+				if configPerNamespace.names != nil {
+					// Consider only the metrics that exist in the configuration
+					exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names)
+					if !exists {
+						continue
+					}
 				}
-				if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
-					continue
+				if configPerNamespace.dimensions != nil {
+					if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
+						continue
+					}
 				}
 				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
 					metricsWithStatistics{
 						cloudwatchMetric: listMetric,
 						statistic:        configPerNamespace.statistics,
 					})
-			} else {
-				// if no metric name and no dimensions given, then keep all listMetricsOutput
-				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
-					metricsWithStatistics{
-						cloudwatchMetric: listMetric,
-						statistic:        configPerNamespace.statistics,
-					})
 			}
 		}
 	}

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go

@@ -1111,6 +1111,7 @@ func TestFilterListMetricsOutput(t *testing.T) {
 	cases := []struct {
 		title                        string
 		listMetricsOutput            []cloudwatchtypes.Metric
+		namespace                    string
 		namespaceDetails             []namespaceDetail
 		filteredMetricWithStatsTotal []metricsWithStatistics
 	}{
@@ -1139,6 +1140,7 @@ func TestFilterListMetricsOutput(t *testing.T) {
 					Namespace:  awssdk.String("AWS/EC2"),
 				},
 			},
+			"AWS/EC2",
 			[]namespaceDetail{
 				{
 					resourceTypeFilter: "ec2:instance",
@@ -1190,6 +1192,7 @@ func TestFilterListMetricsOutput(t *testing.T) {
 					Namespace:  awssdk.String("AWS/EC2"),
 				},
 			},
+			"AWS/EC2",
 			[]namespaceDetail{
 				{
 					names:              []string{"CPUUtilization"},
@@ -1214,7 +1217,7 @@ func TestFilterListMetricsOutput(t *testing.T) {
 	}
 	for _, c := range cases {
 		t.Run(c.title, func(t *testing.T) {
-			output := filterListMetricsOutput(c.listMetricsOutput, c.namespaceDetails)
+			output := filterListMetricsOutput(c.listMetricsOutput, c.namespace, c.namespaceDetails)
 			assert.Equal(t, c.filteredMetricWithStatsTotal, output)
 		})
 	}