From 50c6ae5f3a96d5f1d3d1c016b67e0ac72001701c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 1 Nov 2022 08:49:30 -0600 Subject: [PATCH] [AWS] Add tags to events based on parsed identifier (#33472) --- CHANGELOG.next.asciidoc | 1 + .../module/aws/cloudwatch/cloudwatch.go | 63 +++++++++---------- .../module/aws/cloudwatch/cloudwatch_test.go | 6 +- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 26ffd3ae102..b17a6582cbd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add GCP CloudSQL region filter {pull}32943[32943] - Fix logstash cgroup mappings {pull}33131[33131] - Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263] +- Add tags to events based on parsed identifier. {pull}33472[33472] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 8209f5a6f5f..80894a4b5d1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -565,19 +565,26 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient identifierValue := labels[identifierValueIdx] uniqueIdentifierValue := *output.Label + fmt.Sprint("-", valI) - if _, ok := events[uniqueIdentifierValue]; !ok { - // when tagsFilter is not empty but no entry in - // resourceTagMap for this identifier, do not initialize - // an event for this identifier. - if len(tagsFilter) != 0 && resourceTagMap[identifierValue] == nil { - continue - } - events[uniqueIdentifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) - } - events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) // add tags to event based on identifierValue - insertTags(events, uniqueIdentifierValue, identifierValue, resourceTagMap) + // Check if identifier includes dimensionSeparator (comma in this case), + // split the identifier and check for each sub-identifier. + // For example, identifier might be [storageType, s3BucketName]. + // And tags are only store under s3BucketName in resourceTagMap. + subIdentifiers := strings.Split(identifierValue, dimensionSeparator) + for _, subIdentifier := range subIdentifiers { + if _, ok := events[uniqueIdentifierValue]; !ok { + // when tagsFilter is not empty but no entry in + // resourceTagMap for this identifier, do not initialize + // an event for this identifier. + if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { + continue + } + events[uniqueIdentifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) + } + events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) + insertTags(events, uniqueIdentifierValue, subIdentifier, resourceTagMap) + } } } } @@ -617,28 +624,20 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } -func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { - // Check if identifier includes dimensionSeparator (comma in this case), - // split the identifier and check for each sub-identifier. - // For example, identifier might be [storageType, s3BucketName]. - // And tags are only store under s3BucketName in resourceTagMap. - subIdentifiers := strings.Split(identifier, dimensionSeparator) - for _, v := range subIdentifiers { - tags := resourceTagMap[v] - // some metric dimension values are arn format, eg: AWS/DDOS namespace metric - if len(tags) == 0 && strings.HasPrefix(v, "arn:") { - resourceID, err := aws.FindShortIdentifierFromARN(v) - if err == nil { - tags = resourceTagMap[resourceID] - } +func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { + tags := resourceTagMap[subIdentifier] + // some metric dimension values are arn format, eg: AWS/DDOS namespace metric + if len(tags) == 0 && strings.HasPrefix(subIdentifier, "arn:") { + resourceID, err := aws.FindShortIdentifierFromARN(subIdentifier) + if err == nil { + tags = resourceTagMap[resourceID] } - if len(tags) != 0 { - // By default, replace dot "." using underscore "_" for tag keys. - // Note: tag values are not dedotted. - for _, tag := range tags { - _, _ = events[uniqueIdentifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) - } - continue + } + if len(tags) != 0 { + // By default, replace dot "." using underscore "_" for tag keys. + // Note: tag values are not dedotted. + for _, tag := range tags { + _, _ = events[uniqueIdentifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) } } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 3a4f0104aff..2d3cb912775 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -10,6 +10,7 @@ package cloudwatch import ( "context" "errors" + "strings" "testing" "time" @@ -1646,7 +1647,10 @@ func TestInsertTags(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - insertTags(events, c.identifier, c.identifier, resourceTagMap) + subIdentifiers := strings.Split(c.identifier, dimensionSeparator) + for _, subIdentifier := range subIdentifiers { + insertTags(events, c.identifier, subIdentifier, resourceTagMap) + } value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) assert.Equal(t, c.expectedTagValue, value)