[AWS Cloudwatch] Fix Tags for APIGW

CHANGELOG.next.asciidoc

@@ -343,6 +343,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new metricset cluster for the vSphere module. {pull}40536[40536]
 - Add new metricset network for the vSphere module. {pull}40559[40559]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
+- Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway resources {pull}40755[40755]
 - Add new metricset datastorecluster for vSphere module. {pull}40634[40634]
 - Add support for new metrics in datastorecluster metricset. {pull}40694[40694]
 - Add new metrics for the vSphere Virtualmachine metricset. {pull}40485[40485]

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 // indirect
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/aws/aws-lambda-go v1.44.0
-	github.com/aws/aws-sdk-go-v2 v1.30.4
+	github.com/aws/aws-sdk-go-v2 v1.30.5
 	github.com/aws/aws-sdk-go-v2/config v1.27.29
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.29
 	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5
@@ -179,8 +179,11 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.24
 	github.com/apache/arrow/go/v14 v14.0.2
+	github.com/aws/aws-sdk-go v1.54.19
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13
+	github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8
+	github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8
 	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5
 	github.com/aws/aws-sdk-go-v2/service/health v1.26.4
 	github.com/aws/aws-sdk-go-v2/service/kinesis v1.29.5
@@ -255,8 +258,8 @@ require (
 	github.com/apache/thrift v0.19.0 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect

go.sum

@@ -158,9 +158,11 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-lambda-go v1.44.0 h1:Xp9PANXKsSJ23IhE4ths592uWTCEewswPhSH9qpAuQQ=
 github.com/aws/aws-lambda-go v1.44.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
+github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
+github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
-github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
-github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
+github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
 github.com/aws/aws-sdk-go-v2/config v1.27.29 h1:+ZPKb3u9Up4KZWLGTtpTmC5T3XmRD1ZQ8XQjRCHUvJw=
@@ -171,14 +173,18 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJ
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13 h1:X8EeaOjl91c8sP14NG8EHx5ZxXLJg0tHDp+KQSghp28=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13/go.mod h1:kEI/h2bETfm09LSd7xEEH2qcU1cd//+5HH4Le7p9JgY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY=
+github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8 h1:CgEyY7gfTf7lHYcCi7+w6jJ1XQBugjpadtsuN3TGxdQ=
+github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8/go.mod h1:z99ur4Ha5540t8hb5XtqV/UMOnEoEZK22lhr5ZBS0zw=
+github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8 h1:SWBNBbVbThg5Hdi3hWbVaDFjV/OyPbuqZLu4N+mj/Es=
+github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8/go.mod h1:lz2IT8gzzSwao0Pa6uMSdCIPsprmgCkW83q6sHGZFDw=
 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5 h1:YeTVIy7cJLeahs7K0jQGDGAd1YYND/to/z8N3kqZBhY=
 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5/go.mod h1:y45SdA9v+dLlweaqwAQMoFeXqdRvgwevafa2X8iTqZQ=
 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5 h1:/YvqO1j75i4leoV+Z3a5s/dAlEszf2wTKBW8jc3Gd4s=

metricbeat/docs/modules/aws.asciidoc

@@ -146,6 +146,25 @@ Enforces the use of FIPS service endpoints. See <<aws-credentials-config,AWS cre
     - ec2
 ----
 
+* *apigateway_max_results*
+
+This configuration works together with AWS/APIGateway namespace. It defines the maximum number of returned results per page. The default value is 25 and the maximum value is 500.
+See https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]#GetRestApisInput.Limit[GetRestApisInput.Limit]
+
+[source,yaml]
+----
+- module: aws
+      period: 10s
+      regions:
+        - us-east-1
+      metricsets:
+        - cloudwatch
+      metrics:
+        - namespace: "AWS/ApiGateway"
+          resource_type: "apigateway:restapis"
+          apigateway_max_results: 40
+----
+
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -134,6 +134,25 @@ Enforces the use of FIPS service endpoints. See <<aws-credentials-config,AWS cre
     - ec2
 ----
 
+* *apigateway_max_results*
+
+This configuration works together with AWS/APIGateway namespace. It defines the maximum number of returned results per page. The default value is 25 and the maximum value is 500.
+See https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]#GetRestApisInput.Limit[GetRestApisInput.Limit]
+
+[source,yaml]
+----
+- module: aws
+      period: 10s
+      regions:
+        - us-east-1
+      metricsets:
+        - cloudwatch
+      metrics:
+        - namespace: "AWS/ApiGateway"
+          resource_type: "apigateway:restapis"
+          apigateway_max_results: 40
+----
+
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]

x-pack/metricbeat/module/aws/aws.go

@@ -36,6 +36,7 @@ type Config struct {
 	AWSConfig             awscommon.ConfigAWS `config:",inline"`
 	TagsFilter            []Tag               `config:"tags_filter"`
 	IncludeLinkedAccounts *bool               `config:"include_linked_accounts"`
+	LimitRestAPI          *int32              `config:"apigateway_max_results"`
 	OwningAccount         string              `config:"owning_account"`
 }
 
@@ -207,6 +208,18 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	})
 	metricSet.MonitoringAccountName = getAccountName(svcIam, base, metricSet)
 
+	//Validate LimitRestAPI value.
+	//The Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500.
+	if config.LimitRestAPI != nil {
+		if *config.LimitRestAPI > 500 {
+			base.Logger().Debug("apigateway_max_results config value can not exceed value 500. Setting apigateway_max_results=500")
+			*config.LimitRestAPI = 500
+		} else if *config.LimitRestAPI <= 0 {
+			base.Logger().Debug("apigateway_max_results config value can not be <=0. Setting apigateway_max_results=25")
+			*config.LimitRestAPI = 25
+		}
+	}
+
 	// Construct MetricSet with a full regions list
 	if config.Regions == nil {
 		svcEC2 := ec2.NewFromConfig(awsConfig, func(o *ec2.Options) {

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -6,12 +6,15 @@
 
 import (
 	"fmt"
+	"maps"
 	"reflect"
 	"strconv"
 	"strings"
 	"time"
 
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/apigateway"
+	"github.com/aws/aws-sdk-go-v2/service/apigatewayv2"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
@@ -23,13 +26,25 @@
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
+const checkns = "AWS/ApiGateway"
+const checkresource_type = "apigateway:restapis"
+
 var (
-	metricsetName          = "cloudwatch"
-	defaultStatistics      = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"}
-	dimensionSeparator     = ","
-	dimensionValueWildcard = "*"
+	metricsetName            = "cloudwatch"
+	defaultStatistics        = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"}
+	dimensionSeparator       = ","
+	dimensionValueWildcard   = "*"
+	checkns_lower            = strings.ToLower(checkns)
+	checkresource_type_lower = strings.ToLower(checkresource_type)
 )
 
+type APIClients struct {
+	CloudWatchClient         *cloudwatch.Client
+	Resourcegroupstaggingapi *resourcegroupstaggingapi.Client
+	Apigateway               *apigateway.Client
+	Apigatewayv2             *apigatewayv2.Client
+}
+
 // init registers the MetricSet with the central registry as soon as the program
 // starts. The New function will be called later to instantiate an instance of
 // the MetricSet for each host defined in the module's configuration. After the
@@ -123,7 +138,8 @@
 	startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency, m.PreviousEndTime)
 	m.PreviousEndTime = endTime
 	m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime)
-
+	// Initialise the map that will be used in case APIGateway api is configured. Infoapi includes Name_of_API:ID_of_API entries
+	infoapi := make(map[string]string)
 	// Check statistic method in config
 	err := m.checkStatistics()
 	if err != nil {
@@ -147,13 +163,12 @@
 			m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
 			beatsConfig := m.MetricSet.AwsConfig.Copy()
 			beatsConfig.Region = regionName
-
-			svcCloudwatch, svcResourceAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
+			APIClients, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
 			if err != nil {
 				m.Logger().Warn("skipping metrics list from region '%s'", regionName)
 			}
 
-			eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime)
+			eventsWithIdentifier, err := m.createEvents(APIClients.CloudWatchClient, APIClients.Resourcegroupstaggingapi, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime)
 			if err != nil {
 				return fmt.Errorf("createEvents failed for region %s: %w", regionName, err)
 			}
@@ -173,14 +188,14 @@
 		beatsConfig := m.MetricSet.AwsConfig.Copy()
 		beatsConfig.Region = regionName
 
-		svcCloudwatch, svcResourceAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
+		APIClients, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
 		if err != nil {
 			m.Logger().Warn("skipping metrics list from region '%s'", regionName, err)
 			continue
 		}
 
 		// retrieve all the details for all the metrics available in the current region
-		listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch)
+		listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, APIClients.CloudWatchClient)
 		if err != nil {
 			m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err)
 		}
@@ -191,14 +206,50 @@
 
 		for namespace, namespaceDetails := range namespaceDetailTotal {
 			m.logger.Debugf("Collected metrics from namespace %s", namespace)
-
 			// filter listMetricsOutput by detailed configuration per each namespace
 			filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespace, namespaceDetails)
 
 			// get resource type filters and tags filters for each namespace
 			resourceTypeTagFilters := constructTagsFilters(namespaceDetails)
 
-			eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime)
+			//Check whether namespace is APIGW
+			if strings.Contains(strings.ToLower(namespace), checkns_lower) {
+				useonlyrest := false
+				if len(resourceTypeTagFilters) == 1 {
+					for key := range resourceTypeTagFilters {
+						if strings.Compare(strings.ToLower(key), checkresource_type_lower) == 0 {
+							useonlyrest = true
+						}
+					}
+				}
+				// inforestapi includes only Rest APIs
+				if useonlyrest {
+					infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI)
+					if err != nil {
+						m.Logger().Errorf("could not get rest apis output: %v", err)
+					}
+				} else {
+					// infoapi  includes only Rest APIs
+					// apiGatewayAPI includes only WebSocket and HTTP APIs
+					infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI)
+					if err != nil {
+						m.Logger().Errorf("could not get rest apis output: %v", err)
+					}
+
+					apiGatewayAPI, err := aws.GetAPIGatewayAPIOutput(APIClients.Apigatewayv2)
+					if err != nil {
+						m.Logger().Errorf("could not get http and websocket apis output: %v", err)
+					}
+					if len(apiGatewayAPI) > 0 {
+						maps.Copy(infoapi, apiGatewayAPI)
+					}
+
+				}
+
+				m.Logger().Debugf("infoapi response: %v", infoapi)
+
+			}
+			eventsWithIdentifier, err := m.createEvents(APIClients.CloudWatchClient, APIClients.Resourcegroupstaggingapi, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime)
 			if err != nil {
 				return fmt.Errorf("createEvents failed for region %s: %w", regionName, err)
 			}
@@ -221,23 +272,32 @@
 }
 
 // createAwsRequiredClients will return the two necessary client instances to do Metric requests to the AWS API
-func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, error) {
+func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (APIClients, error) {
 	m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
 
-	svcCloudwatchClient := cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) {
+	APIClients := APIClients{}
+	APIClients.CloudWatchClient = cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) {
 		if config.AWSConfig.FIPSEnabled {
 			o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled
 		}
 
 	})
 
-	svcResourceAPIClient := resourcegroupstaggingapi.NewFromConfig(beatsConfig, func(o *resourcegroupstaggingapi.Options) {
+	APIClients.Resourcegroupstaggingapi = resourcegroupstaggingapi.NewFromConfig(beatsConfig, func(o *resourcegroupstaggingapi.Options) {
 		if config.AWSConfig.FIPSEnabled {
 			o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled
 		}
 	})
 
-	return svcCloudwatchClient, svcResourceAPIClient, nil
+	APIClients.Apigateway = apigateway.NewFromConfig(beatsConfig, func(o *apigateway.Options) {
+
+	})
+
+	APIClients.Apigatewayv2 = apigatewayv2.NewFromConfig(beatsConfig, func(o *apigatewayv2.Options) {
+
+	})
+
+	return APIClients, nil
 }
 
 // filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match
@@ -458,7 +518,7 @@
 	return event
 }
 
-func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) {
+func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, infoAPImap map[string]string, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) {
 	// Initialize events for each identifier.
 	events := make(map[string]mb.Event)
 
@@ -568,6 +628,13 @@
 				// And tags are only store under s3BucketName in resourceTagMap.
 				subIdentifiers := strings.Split(identifierValue, dimensionSeparator)
 				for _, subIdentifier := range subIdentifiers {
+
+					if len(infoAPImap) > 0 { // If infoAPImap includes data
+						if valAPIName, ok := infoAPImap[subIdentifier]; ok {
+							subIdentifier = valAPIName
+						}
+					}
+
 					if _, ok := events[uniqueIdentifierValue]; !ok {
 						// when tagsFilter is not empty but no entry in
 						// resourceTagMap for this identifier, do not initialize