Skip to content

Commit

Permalink
[AWS Cloudwatch] Changed module to call ListMetrics API only once per…
Browse files Browse the repository at this point in the history
… region (#34055)

* Changed cloudwatch module to call ListMetrics API only once per region, instead of per AWS namespace
* Refactored filterListMetricsOutput method
  • Loading branch information
girodav authored Dec 19, 2022
1 parent edce024 commit 5fd0041
Show file tree
Hide file tree
Showing 5 changed files with 156 additions and 387 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
- Add support for multiple regions in GCP {pull}32964[32964]
- Add GCP Redis regions support {pull}33728[33728]
- Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763]
- Changed cloudwatch module to call ListMetrics API only once per region, instead of per AWS namespace {pull}34055[34055]

*Packetbeat*

Expand Down
2 changes: 1 addition & 1 deletion metricbeat/docs/modules/aws.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
| IAM ListAccountAliases | 1 | Once on startup
| STS GetCallerIdentity | 1 | Once on startup
| EC2 DescribeRegions| 1 | Once on startup
| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per namespace per collection period
| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
|===
`billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.
Expand Down
2 changes: 1 addition & 1 deletion x-pack/metricbeat/module/aws/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
| IAM ListAccountAliases | 1 | Once on startup
| STS GetCallerIdentity | 1 | Once on startup
| EC2 DescribeRegions| 1 | Once on startup
| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per namespace per collection period
| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
|===
`billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.
Expand Down
77 changes: 28 additions & 49 deletions x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,23 +178,26 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {

svcCloudwatch, svcResourceAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
if err != nil {
m.Logger().Warn("skipping metrics list from region '%s'", regionName)
m.Logger().Warn("skipping metrics list from region '%s'", regionName, err)
continue
}
for namespace, namespaceDetails := range namespaceDetailTotal {
m.logger.Debugf("Collected metrics from namespace %s", namespace)

listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, svcCloudwatch)
if err != nil {
m.logger.Info(err.Error())
continue
}
// retrieve all the details for all the metrics available in the current region
listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, svcCloudwatch)
if err != nil {
m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err)
}

if len(listMetricsOutput) == 0 {
continue
}
if len(listMetricsOutput) == 0 {
continue
}

for namespace, namespaceDetails := range namespaceDetailTotal {
m.logger.Debugf("Collected metrics from namespace %s", namespace)

// filter listMetricsOutput by detailed configuration per each namespace
filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespaceDetails)
filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespace, namespaceDetails)

// get resource type filters and tags filters for each namespace
resourceTypeTagFilters := constructTagsFilters(namespaceDetails)

Expand Down Expand Up @@ -240,52 +243,28 @@ func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionNa
}

// filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match
func filterListMetricsOutput(listMetricsOutput []types.Metric, namespaceDetails []namespaceDetail) []metricsWithStatistics {
func filterListMetricsOutput(listMetricsOutput []types.Metric, namespace string, namespaceDetails []namespaceDetail) []metricsWithStatistics {
var filteredMetricWithStatsTotal []metricsWithStatistics
for _, listMetric := range listMetricsOutput {
for _, configPerNamespace := range namespaceDetails {
if configPerNamespace.names != nil && configPerNamespace.dimensions == nil {
// if metric names are given in config but no dimensions, filter
// out the metrics with other names
if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
continue
}
filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
metricsWithStatistics{
cloudwatchMetric: listMetric,
statistic: configPerNamespace.statistics,
})

} else if configPerNamespace.names == nil && configPerNamespace.dimensions != nil {
// if metric names are not given in config but dimensions are
// given, only keep the metrics with matching dimensions
if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
continue
}
filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
metricsWithStatistics{
cloudwatchMetric: listMetric,
statistic: configPerNamespace.statistics,
})
} else if configPerNamespace.names != nil && configPerNamespace.dimensions != nil {
if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
continue
if *listMetric.Namespace == namespace {
for _, configPerNamespace := range namespaceDetails {
if configPerNamespace.names != nil {
// Consider only the metrics that exist in the configuration
exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names)
if !exists {
continue
}
}
if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
continue
if configPerNamespace.dimensions != nil {
if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
continue
}
}
filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
metricsWithStatistics{
cloudwatchMetric: listMetric,
statistic: configPerNamespace.statistics,
})
} else {
// if no metric name and no dimensions given, then keep all listMetricsOutput
filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
metricsWithStatistics{
cloudwatchMetric: listMetric,
statistic: configPerNamespace.statistics,
})
}
}
}
Expand Down
Loading

0 comments on commit 5fd0041

Please sign in to comment.