[AWS Cloudwatch] Changed module to call ListMetrics API only once per region

CHANGELOG.next.asciidoc

@@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Add support for multiple regions in GCP {pull}32964[32964]
 - Add GCP Redis regions support {pull}33728[33728]
 - Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763]
+- Changed cloudwatch module to call ListMetrics API only once per region, instead of per AWS namespace {pull}34055[34055]
 
 *Packetbeat*
 

metricbeat/docs/modules/aws.asciidoc

@@ -297,7 +297,7 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 | IAM ListAccountAliases | 1 | Once on startup
 | STS GetCallerIdentity | 1 | Once on startup
 | EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per namespace per collection period
+| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
 | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -285,7 +285,7 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 | IAM ListAccountAliases | 1 | Once on startup
 | STS GetCallerIdentity | 1 | Once on startup
 | EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per namespace per collection period
+| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
 | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -178,23 +178,26 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 
 		svcCloudwatch, svcResourceAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
 		if err != nil {
-			m.Logger().Warn("skipping metrics list from region '%s'", regionName)
+			m.Logger().Warn("skipping metrics list from region '%s'", regionName, err)
+			continue
 		}
-		for namespace, namespaceDetails := range namespaceDetailTotal {
-			m.logger.Debugf("Collected metrics from namespace %s", namespace)
 
-			listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, svcCloudwatch)
-			if err != nil {
-				m.logger.Info(err.Error())
-				continue
-			}
+		// retrieve all the details for all the metrics available in the current region
+		listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, svcCloudwatch)
+		if err != nil {
+			m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err)
+		}
 
-			if len(listMetricsOutput) == 0 {
-				continue
-			}
+		if len(listMetricsOutput) == 0 {
+			continue
+		}
+
+		for namespace, namespaceDetails := range namespaceDetailTotal {
+			m.logger.Debugf("Collected metrics from namespace %s", namespace)
 
 			// filter listMetricsOutput by detailed configuration per each namespace
-			filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespaceDetails)
+			filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespace, namespaceDetails)
+
 			// get resource type filters and tags filters for each namespace
 			resourceTypeTagFilters := constructTagsFilters(namespaceDetails)
 
@@ -240,52 +243,28 @@ func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionNa
 }
 
 // filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match
-func filterListMetricsOutput(listMetricsOutput []types.Metric, namespaceDetails []namespaceDetail) []metricsWithStatistics {
+func filterListMetricsOutput(listMetricsOutput []types.Metric, namespace string, namespaceDetails []namespaceDetail) []metricsWithStatistics {
 	var filteredMetricWithStatsTotal []metricsWithStatistics
 	for _, listMetric := range listMetricsOutput {
-		for _, configPerNamespace := range namespaceDetails {
-			if configPerNamespace.names != nil && configPerNamespace.dimensions == nil {
-				// if metric names are given in config but no dimensions, filter
-				// out the metrics with other names
-				if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
-					continue
-				}
-				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
-					metricsWithStatistics{
-						cloudwatchMetric: listMetric,
-						statistic:        configPerNamespace.statistics,
-					})
-
-			} else if configPerNamespace.names == nil && configPerNamespace.dimensions != nil {
-				// if metric names are not given in config but dimensions are
-				// given, only keep the metrics with matching dimensions
-				if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
-					continue
-				}
-				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
-					metricsWithStatistics{
-						cloudwatchMetric: listMetric,
-						statistic:        configPerNamespace.statistics,
-					})
-			} else if configPerNamespace.names != nil && configPerNamespace.dimensions != nil {
-				if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
-					continue
+		if *listMetric.Namespace == namespace {
+			for _, configPerNamespace := range namespaceDetails {
+				if configPerNamespace.names != nil {
+					// Consider only the metrics that exist in the configuration
+					exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names)
+					if !exists {
+						continue
+					}
 				}
-				if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
-					continue
+				if configPerNamespace.dimensions != nil {
+					if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
+						continue
+					}
 				}
 				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
 					metricsWithStatistics{
 						cloudwatchMetric: listMetric,
 						statistic:        configPerNamespace.statistics,
 					})
-			} else {
-				// if no metric name and no dimensions given, then keep all listMetricsOutput
-				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
-					metricsWithStatistics{
-						cloudwatchMetric: listMetric,
-						statistic:        configPerNamespace.statistics,
-					})
 			}
 		}
 	}