diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6d0beada75a..2ef62f6baf7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -171,6 +171,8 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Metricbeat* + +- Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}33166[33166] - Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308] *Packetbeat* diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 868ba92a184..0dd31bf2543 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -49,6 +49,17 @@ or none get collected by Metricbeat. In this case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *data_granularity* + +AWS CloudWatch allows to define the granularity of the returned datapoints, by setting "Period" while querying metrics. +Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDataQuery.html[MetricDataQuery parameters] for more information. + +By default, metricbeat will query CloudWatch setting "Period" to Metricbeat collection period. If you wish to set a custom value for "Period", please specify a `data_granularity` parameter. +By setting `period` and `data_granularity` together, you can control, respectively, how frequently you want your metrics to be collected and how granular they have to be. + +If you are concerned about reducing the cost derived by CloudWatch API calls made by Metricbeat with an extra delay in retrieving metrics as trade off, you may consider setting `data_granularity` and increase Metricbeat collection period. For example, +setting `data_granularity` to your current value for `period`, and doubling the value of `period`, may lead to a 50% savings in terms of GetMetricData API calls cost. + * *endpoint* Most AWS services offer a regional endpoint that can be used to make requests. @@ -69,7 +80,7 @@ For example, if tags parameter is given as `Organization=Engineering` under `Organization` and tag value equals to `Engineering`. In order to filter for different values for the same key, add the values to the value array (see example) -Note: tag filtering only works for metricsets with `resource_type` specified in the +Note: tag filtering only works for metricsets with `resource_type` specified in the metricset-specific configuration. [source,yaml] diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index c446aa95758..c7db5622953 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -37,6 +37,17 @@ or none get collected by Metricbeat. In this case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *data_granularity* + +AWS CloudWatch allows to define the granularity of the returned datapoints, by setting "Period" while querying metrics. +Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDataQuery.html[MetricDataQuery parameters] for more information. + +By default, metricbeat will query CloudWatch setting "Period" to Metricbeat collection period. If you wish to set a custom value for "Period", please specify a `data_granularity` parameter. +By setting `period` and `data_granularity` together, you can control, respectively, how frequently you want your metrics to be collected and how granular they have to be. + +If you are concerned about reducing the cost derived by CloudWatch API calls made by Metricbeat with an extra delay in retrieving metrics as trade off, you may consider setting `data_granularity` and increase Metricbeat collection period. For example, +setting `data_granularity` to your current value for `period`, and doubling the value of `period`, may lead to a 50% savings in terms of GetMetricData API calls cost. + * *endpoint* Most AWS services offer a regional endpoint that can be used to make requests. @@ -57,7 +68,7 @@ For example, if tags parameter is given as `Organization=Engineering` under `Organization` and tag value equals to `Engineering`. In order to filter for different values for the same key, add the values to the value array (see example) -Note: tag filtering only works for metricsets with `resource_type` specified in the +Note: tag filtering only works for metricsets with `resource_type` specified in the metricset-specific configuration. [source,yaml] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 3e286f66560..ea4d24e0240 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -29,24 +29,26 @@ type describeRegionsClient interface { // Config defines all required and optional parameters for aws metricsets type Config struct { - Period time.Duration `config:"period" validate:"nonzero,required"` - Regions []string `config:"regions"` - Latency time.Duration `config:"latency"` - AWSConfig awscommon.ConfigAWS `config:",inline"` - TagsFilter []Tag `config:"tags_filter"` + Period time.Duration `config:"period" validate:"nonzero,required"` + DataGranularity time.Duration `config:"data_granularity"` + Regions []string `config:"regions"` + Latency time.Duration `config:"latency"` + AWSConfig awscommon.ConfigAWS `config:",inline"` + TagsFilter []Tag `config:"tags_filter"` } // MetricSet is the base metricset for all aws metricsets type MetricSet struct { mb.BaseMetricSet - RegionsList []string - Endpoint string - Period time.Duration - Latency time.Duration - AwsConfig *awssdk.Config - AccountName string - AccountID string - TagsFilter []Tag + RegionsList []string + Endpoint string + Period time.Duration + DataGranularity time.Duration + Latency time.Duration + AwsConfig *awssdk.Config + AccountName string + AccountID string + TagsFilter []Tag } // Tag holds a configuration specific for ec2 and cloudwatch metricset. @@ -91,16 +93,25 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint) + if config.DataGranularity > config.Period { + return nil, fmt.Errorf("Data Granularity cannot be larger than the period") + } + + if config.DataGranularity == 0 { + config.DataGranularity = config.Period + } metricSet := MetricSet{ - BaseMetricSet: base, - Period: config.Period, - Latency: config.Latency, - AwsConfig: &awsConfig, - TagsFilter: config.TagsFilter, - Endpoint: config.AWSConfig.Endpoint, + BaseMetricSet: base, + Period: config.Period, + DataGranularity: config.DataGranularity, + Latency: config.Latency, + AwsConfig: &awsConfig, + TagsFilter: config.TagsFilter, + Endpoint: config.AWSConfig.Endpoint, } base.Logger().Debug("Metricset level config for period: ", metricSet.Period) + base.Logger().Debug("Metricset level config for data granularity: ", metricSet.DataGranularity) base.Logger().Debug("Metricset level config for tags filter: ", metricSet.TagsFilter) base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset") diff --git a/x-pack/metricbeat/module/aws/billing/billing.go b/x-pack/metricbeat/module/aws/billing/billing.go index 37f88deccd9..833e993bee2 100644 --- a/x-pack/metricbeat/module/aws/billing/billing.go +++ b/x-pack/metricbeat/module/aws/billing/billing.go @@ -181,7 +181,7 @@ func (m *MetricSet) getCloudWatchBillingMetrics( return events } - metricDataQueriesTotal := constructMetricQueries(listMetricsOutput, m.Period) + metricDataQueriesTotal := constructMetricQueries(listMetricsOutput, m.DataGranularity) metricDataOutput, err := aws.GetMetricDataResults(metricDataQueriesTotal, svcCloudwatch, startTime, endTime) if err != nil { err = fmt.Errorf("aws GetMetricDataResults failed with %w, skipping region %s", err, regionName) @@ -189,22 +189,15 @@ func (m *MetricSet) getCloudWatchBillingMetrics( return nil } - // Find a timestamp for all metrics in output - timestamp := aws.FindTimestamp(metricDataOutput) - if timestamp.IsZero() { - return nil - } - for _, output := range metricDataOutput { if len(output.Values) == 0 { continue } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { + for valI, metricDataResultValue := range output.Values { labels := strings.Split(*output.Label, labelSeparator) - event := aws.InitEvent("", m.AccountName, m.AccountID, timestamp) - _, _ = event.MetricSetFields.Put(labels[0], output.Values[timestampIdx]) + event := aws.InitEvent("", m.AccountName, m.AccountID, output.Timestamps[valI]) + _, _ = event.MetricSetFields.Put(labels[0], metricDataResultValue) i := 1 for i < len(labels)-1 { @@ -345,11 +338,11 @@ func (m *MetricSet) addCostMetrics(metrics map[string]costexplorertypes.MetricVa return event } -func constructMetricQueries(listMetricsOutput []types.Metric, period time.Duration) []types.MetricDataQuery { +func constructMetricQueries(listMetricsOutput []types.Metric, dataGranularity time.Duration) []types.MetricDataQuery { var metricDataQueries []types.MetricDataQuery metricDataQueryEmpty := types.MetricDataQuery{} for i, listMetric := range listMetricsOutput { - metricDataQuery := createMetricDataQuery(listMetric, i, period) + metricDataQuery := createMetricDataQuery(listMetric, i, dataGranularity) if metricDataQuery == metricDataQueryEmpty { continue } @@ -358,9 +351,9 @@ func constructMetricQueries(listMetricsOutput []types.Metric, period time.Durati return metricDataQueries } -func createMetricDataQuery(metric types.Metric, index int, period time.Duration) types.MetricDataQuery { +func createMetricDataQuery(metric types.Metric, index int, dataGranularity time.Duration) types.MetricDataQuery { statistic := "Maximum" - periodInSeconds := int32(period.Seconds()) + dataGranularityInSeconds := int32(dataGranularity.Seconds()) id := metricsetName + strconv.Itoa(index) metricDims := metric.Dimensions metricName := *metric.MetricName @@ -373,7 +366,7 @@ func createMetricDataQuery(metric types.Metric, index int, period time.Duration) return types.MetricDataQuery{ Id: &id, MetricStat: &types.MetricStat{ - Period: &periodInSeconds, + Period: &dataGranularityInSeconds, Stat: &statistic, Metric: &metric, }, diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 7f7ac5a2d1d..8209f5a6f5f 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -378,20 +378,20 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n return listMetricDetailTotal, namespaceDetailTotal } -func createMetricDataQueries(listMetricsTotal []metricsWithStatistics, period time.Duration) []types.MetricDataQuery { +func createMetricDataQueries(listMetricsTotal []metricsWithStatistics, dataGranularity time.Duration) []types.MetricDataQuery { var metricDataQueries []types.MetricDataQuery for i, listMetric := range listMetricsTotal { for j, statistic := range listMetric.statistic { stat := statistic metric := listMetric.cloudwatchMetric label := constructLabel(listMetric.cloudwatchMetric, statistic) - periodInSec := int32(period.Seconds()) + dataGranularityInSec := int32(dataGranularity.Seconds()) id := "cw" + strconv.Itoa(i) + "stats" + strconv.Itoa(j) metricDataQueries = append(metricDataQueries, types.MetricDataQuery{ Id: &id, MetricStat: &types.MetricStat{ - Period: &periodInSec, + Period: &dataGranularityInSec, Stat: &stat, Metric: &metric, }, @@ -473,10 +473,10 @@ func insertRootFields(event mb.Event, metricValue float64, labels []string) mb.E func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) { // Initialize events for each identifier. - events := map[string]mb.Event{} + events := make(map[string]mb.Event) // Construct metricDataQueries - metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.Period) + metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.DataGranularity) m.logger.Debugf("Number of MetricDataQueries = %d", len(metricDataQueries)) if len(metricDataQueries) == 0 { return events, nil @@ -489,37 +489,29 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient return events, fmt.Errorf("getMetricDataResults failed: %w", err) } - // Find a timestamp for all metrics in output - timestamp := aws.FindTimestamp(metricDataResults) - if timestamp.IsZero() { - return nil, nil - } - // Create events when there is no tags_filter or resource_type specified. if len(resourceTypeTagFilters) == 0 { for _, metricDataResult := range metricDataResults { if len(metricDataResult.Values) == 0 { continue } - - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, metricDataResult.Timestamps) - if exists { - labels := strings.Split(*metricDataResult.Label, labelSeparator) + labels := strings.Split(*metricDataResult.Label, labelSeparator) + for valI, metricDataResultValue := range metricDataResult.Values { if len(labels) != 5 { - // when there is no identifier value in label, use region+accountID+namespace instead - identifier := regionName + m.AccountID + labels[namespaceIdx] + // when there is no identifier value in label, use region+accountID+label+index instead + identifier := regionName + m.AccountID + *metricDataResult.Label + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) } - events[identifier] = insertRootFields(events[identifier], metricDataResult.Values[timestampIdx], labels) + events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } - identifierValue := labels[identifierValueIdx] + identifierValue := *metricDataResult.Label + fmt.Sprint("-", valI) if _, ok := events[identifierValue]; !ok { - events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) } - events[identifierValue] = insertRootFields(events[identifierValue], metricDataResult.Values[timestampIdx], labels) + events[identifierValue] = insertRootFields(events[identifierValue], metricDataResultValue, labels) } } return events, nil @@ -554,38 +546,38 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient continue } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - labels := strings.Split(*output.Label, labelSeparator) + labels := strings.Split(*output.Label, labelSeparator) + for valI, metricDataResultValue := range output.Values { if len(labels) != 5 { // if there is no tag in labels but there is a tagsFilter, then no event should be reported. if len(tagsFilter) != 0 { continue } - // when there is no identifier value in label, use region+accountID+namespace instead - identifier := regionName + m.AccountID + labels[namespaceIdx] + // when there is no identifier value in label, use region+accountID+labels instead + identifier := regionName + m.AccountID + *output.Label + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) } - events[identifier] = insertRootFields(events[identifier], output.Values[timestampIdx], labels) + events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } identifierValue := labels[identifierValueIdx] - if _, ok := events[identifierValue]; !ok { + uniqueIdentifierValue := *output.Label + fmt.Sprint("-", valI) + if _, ok := events[uniqueIdentifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize // an event for this identifier. if len(tagsFilter) != 0 && resourceTagMap[identifierValue] == nil { continue } - events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[uniqueIdentifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) } - events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) + events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) // add tags to event based on identifierValue - insertTags(events, identifierValue, resourceTagMap) + insertTags(events, uniqueIdentifierValue, identifierValue, resourceTagMap) } } } @@ -625,7 +617,7 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } -func insertTags(events map[string]mb.Event, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { +func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { // Check if identifier includes dimensionSeparator (comma in this case), // split the identifier and check for each sub-identifier. // For example, identifier might be [storageType, s3BucketName]. @@ -644,7 +636,7 @@ func insertTags(events map[string]mb.Event, identifier string, resourceTagMap ma // By default, replace dot "." using underscore "_" for tag keys. // Note: tag values are not dedotted. for _, tag := range tags { - _, _ = events[identifier].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) + _, _ = events[uniqueIdentifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) } continue } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 5417b8323cc..3a4f0104aff 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1344,6 +1344,33 @@ func (m *MockCloudWatchClientWithoutDim) GetMetricData(context.Context, *cloudwa }, nil } +// MockCloudWatchClientWithDataGranularity struct is used for unit tests. +type MockCloudWatchClientWithDataGranularity struct{} + +// GetMetricData implements cloudwatch.GetMetricDataAPIClient. +func (m *MockCloudWatchClientWithDataGranularity) GetMetricData(context.Context, *cloudwatch.GetMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.GetMetricDataOutput, error) { + emptyString := "" + return &cloudwatch.GetMetricDataOutput{ + Messages: nil, + MetricDataResults: []cloudwatchtypes.MetricDataResult{ + { + Id: &id1, + Label: &label3, + Values: []float64{value1, value1}, + Timestamps: []time.Time{timestamp, timestamp}, + }, + { + Id: &id2, + Label: &label4, + Values: []float64{value2, value2}, + Timestamps: []time.Time{timestamp, timestamp}, + }, + }, + NextToken: &emptyString, + ResultMetadata: middleware.Metadata{}, + }, nil +} + // MockResourceGroupsTaggingClient is used for unit tests. type MockResourceGroupsTaggingClient struct{} @@ -1396,12 +1423,13 @@ func TestCreateEventsWithIdentifier(t *testing.T) { events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) + assert.Equal(t, 2, len(events)) - metricValue, err := events["i-1"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + metricValue, err := events[label1+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") assert.NoError(t, err) assert.Equal(t, value1, metricValue) - dimension, err := events["i-1"].RootFields.GetValue("aws.dimensions.InstanceId") + dimension, err := events[label2+"-0"].RootFields.GetValue("aws.dimensions.InstanceId") assert.NoError(t, err) assert.Equal(t, instanceID1, dimension) } @@ -1437,20 +1465,67 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) - expectedID := regionName + accountID + namespace - metricValue, err := events[expectedID].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + expectedID := regionName + accountID + metricValue, err := events[expectedID+label3+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") assert.NoError(t, err) assert.Equal(t, value1, metricValue) - dimension, err := events[expectedID].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + dimension, err := events[expectedID+label4+"-0"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") assert.NoError(t, err) assert.Equal(t, value2, dimension) } +func TestCreateEventsWithDataGranularity(t *testing.T) { + m := MetricSet{} + m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} + m.MetricSet = &aws.MetricSet{Period: 10, AccountID: accountID, DataGranularity: 5} + m.logger = logp.NewLogger("test") + + mockTaggingSvc := &MockResourceGroupsTaggingClient{} + mockCloudwatchSvc := &MockCloudWatchClientWithDataGranularity{} + listMetricWithStatsTotal := []metricsWithStatistics{ + { + cloudwatchtypes.Metric{ + MetricName: awssdk.String("CPUUtilization"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, + { + cloudwatchtypes.Metric{ + MetricName: awssdk.String("DiskReadOps"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, + } + + resourceTypeTagFilters := map[string][]aws.Tag{} + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) + + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + assert.NoError(t, err) + + expectedID := regionName + accountID + metricValue, err := events[expectedID+label3+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + assert.NoError(t, err) + metricValue1, err := events[expectedID+label3+"-1"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + assert.NoError(t, err) + metricValue2, err := events[expectedID+label4+"-0"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + assert.NoError(t, err) + metricValue3, err := events[expectedID+label4+"-1"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + assert.NoError(t, err) + assert.Equal(t, value1, metricValue) + assert.Equal(t, value1, metricValue1) + assert.Equal(t, value2, metricValue2) + assert.Equal(t, value2, metricValue3) + assert.Equal(t, 4, len(events)) +} + func TestCreateEventsWithTagsFilter(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} - m.MetricSet = &aws.MetricSet{Period: 5} + m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID} m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} @@ -1467,6 +1542,17 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { }, []string{"Average"}, }, + { + cloudwatchtypes.Metric{ + Dimensions: []cloudwatchtypes.Dimension{{ + Name: awssdk.String("InstanceId"), + Value: awssdk.String("i-1"), + }}, + MetricName: awssdk.String("DiskReadOps"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, } // Check that the event is created when the tag filter matches @@ -1481,7 +1567,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) - assert.Equal(t, 1, len(events)) + assert.Equal(t, 2, len(events)) // Specify a tag filter that does not match the tag for i-1 resourceTypeTagFilters["ec2:instance"] = []aws.Tag{ @@ -1560,7 +1646,7 @@ func TestInsertTags(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - insertTags(events, c.identifier, resourceTagMap) + insertTags(events, c.identifier, c.identifier, resourceTagMap) value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) assert.Equal(t, c.expectedTagValue, value) @@ -1627,6 +1713,13 @@ func TestCreateEventsTimestamp(t *testing.T) { }, []string{"Average"}, }, + { + cloudwatchtypes.Metric{ + MetricName: awssdk.String("DiskReadOps"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, } resourceTypeTagFilters := map[string][]aws.Tag{} @@ -1636,7 +1729,8 @@ func TestCreateEventsTimestamp(t *testing.T) { resGroupTaggingClientMock := &MockResourceGroupsTaggingClient{} events, err := m.createEvents(cloudwatchMock, resGroupTaggingClientMock, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) - assert.Equal(t, timestamp, events[regionName+accountID+namespace].Timestamp) + assert.Equal(t, timestamp, events[regionName+accountID+label3+"-0"].Timestamp) + assert.Equal(t, timestamp, events[regionName+accountID+label4+"-0"].Timestamp) } func TestGetStartTimeEndTime(t *testing.T) { diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index d9810a8cae6..37ef6803ea0 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -118,51 +118,6 @@ func CheckTimestampInArray(timestamp time.Time, timestampArray []time.Time) (boo return false, -1 } -// FindTimestamp function checks MetricDataResults and find the timestamp to collect metrics from. -// For example, MetricDataResults might look like: -// -// metricDataResults = [{ -// Id: "sqs0", -// Label: "testName SentMessageSize", -// StatusCode: Complete, -// Timestamps: [2019-03-11 17:45:00 +0000 UTC], -// Values: [981] -// } { -// -// Id: "sqs1", -// Label: "testName NumberOfMessagesSent", -// StatusCode: Complete, -// Timestamps: [2019-03-11 17:45:00 +0000 UTC,2019-03-11 17:40:00 +0000 UTC], -// Values: [0.5,0] -// }] -// -// This case, we are collecting values for both metrics from timestamp 2019-03-11 17:45:00 +0000 UTC. -func FindTimestamp(getMetricDataResults []types.MetricDataResult) time.Time { - timestamp := time.Time{} - for _, output := range getMetricDataResults { - // When there are outputs with one timestamp, use this timestamp. - if output.Timestamps != nil && len(output.Timestamps) == 1 { - // Use the first timestamp from Timestamps field to collect the latest data. - timestamp = output.Timestamps[0] - return timestamp - } - } - - // When there is no output with one timestamp, use the latest timestamp from timestamp list. - if timestamp.IsZero() { - for _, output := range getMetricDataResults { - // When there are outputs with one timestamp, use this timestamp - if output.Timestamps != nil && len(output.Timestamps) > 1 { - // Example Timestamps: [2019-03-11 17:36:00 +0000 UTC,2019-03-11 17:31:00 +0000 UTC] - timestamp = output.Timestamps[0] - return timestamp - } - } - } - - return timestamp -} - // GetResourcesTags function queries AWS resource groupings tagging API // to get a resource tag mapping with specific resource type filters func GetResourcesTags(svc resourcegroupstaggingapi.GetResourcesAPIClient, resourceTypeFilters []string) (map[string][]resourcegroupstaggingapitypes.Tag, error) { diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index d1dad2c756b..ebaa3121429 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -260,81 +260,6 @@ func TestCheckTimestampInArray(t *testing.T) { } } -func TestFindTimestamp(t *testing.T) { - timestamp1 := time.Now() - timestamp2 := timestamp1.Add(5 * time.Minute) - cases := []struct { - getMetricDataResults []cloudwatchtypes.MetricDataResult - expectedTimestamp time.Time - }{ - { - getMetricDataResults: []cloudwatchtypes.MetricDataResult{ - { - Id: &id1, - Label: &label1, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1, timestamp2}, - Values: []float64{0, 1}, - }, - { - Id: &id2, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1}, - Values: []float64{2, 3}, - }, - }, - expectedTimestamp: timestamp1, - }, - { - getMetricDataResults: []cloudwatchtypes.MetricDataResult{ - { - Id: &id1, - Label: &label1, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1, timestamp2}, - Values: []float64{0, 1}, - }, - { - Id: &id2, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - }, - }, - expectedTimestamp: timestamp1, - }, - { - getMetricDataResults: []cloudwatchtypes.MetricDataResult{ - { - Id: &id1, - Label: &label1, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1, timestamp2}, - Values: []float64{0, 1}, - }, - { - Id: &id2, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - }, - { - Id: &id3, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp2}, - Values: []float64{2, 3}, - }, - }, - expectedTimestamp: timestamp2, - }, - } - - for _, c := range cases { - outputTimestamp := FindTimestamp(c.getMetricDataResults) - assert.Equal(t, c.expectedTimestamp, outputTimestamp) - } -} - func TestFindIdentifierFromARN(t *testing.T) { cases := []struct { resourceARN string