From 15a0cc45bd68ea3ec2261413bd708d904b55acd8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 1 Nov 2022 08:49:30 -0600 Subject: [PATCH 1/3] [AWS] Add tags to events based on parsed identifier (#33472) (cherry picked from commit 50c6ae5f3a96d5f1d3d1c016b67e0ac72001701c) # Conflicts: # x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go # x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go --- CHANGELOG.next.asciidoc | 9 ++++ .../module/aws/cloudwatch/cloudwatch.go | 41 +++++++++++++++++++ .../module/aws/cloudwatch/cloudwatch_test.go | 8 ++++ 3 files changed, 58 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 30d6d0faa17..5516c2a60f5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -61,6 +61,15 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Metricbeat* +- Fix GCP storage field naming {pull}32806[32806] +- in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305] +- Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] +- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032] +- Change max query size for GetMetricData API to 500 and add RecentlyActive for ListMetrics API call {pull}33105[33105] +- Add GCP CloudSQL region filter {pull}32943[32943] +- Fix logstash cgroup mappings {pull}33131[33131] +- Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263] +- Add tags to events based on parsed identifier. {pull}33472[33472] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 7f7ac5a2d1d..1a3418691bf 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -573,6 +573,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient } identifierValue := labels[identifierValueIdx] +<<<<<<< HEAD if _, ok := events[identifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize @@ -586,6 +587,29 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient // add tags to event based on identifierValue insertTags(events, identifierValue, resourceTagMap) +======= + uniqueIdentifierValue := *output.Label + fmt.Sprint("-", valI) + + // add tags to event based on identifierValue + // Check if identifier includes dimensionSeparator (comma in this case), + // split the identifier and check for each sub-identifier. + // For example, identifier might be [storageType, s3BucketName]. + // And tags are only store under s3BucketName in resourceTagMap. + subIdentifiers := strings.Split(identifierValue, dimensionSeparator) + for _, subIdentifier := range subIdentifiers { + if _, ok := events[uniqueIdentifierValue]; !ok { + // when tagsFilter is not empty but no entry in + // resourceTagMap for this identifier, do not initialize + // an event for this identifier. + if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { + continue + } + events[uniqueIdentifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) + } + events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) + insertTags(events, uniqueIdentifierValue, subIdentifier, resourceTagMap) + } +>>>>>>> 50c6ae5f3a ([AWS] Add tags to events based on parsed identifier (#33472)) } } } @@ -625,6 +649,7 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } +<<<<<<< HEAD func insertTags(events map[string]mb.Event, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { // Check if identifier includes dimensionSeparator (comma in this case), // split the identifier and check for each sub-identifier. @@ -647,6 +672,22 @@ func insertTags(events map[string]mb.Event, identifier string, resourceTagMap ma _, _ = events[identifier].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) } continue +======= +func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { + tags := resourceTagMap[subIdentifier] + // some metric dimension values are arn format, eg: AWS/DDOS namespace metric + if len(tags) == 0 && strings.HasPrefix(subIdentifier, "arn:") { + resourceID, err := aws.FindShortIdentifierFromARN(subIdentifier) + if err == nil { + tags = resourceTagMap[resourceID] + } + } + if len(tags) != 0 { + // By default, replace dot "." using underscore "_" for tag keys. + // Note: tag values are not dedotted. + for _, tag := range tags { + _, _ = events[uniqueIdentifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) +>>>>>>> 50c6ae5f3a ([AWS] Add tags to events based on parsed identifier (#33472)) } } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 5417b8323cc..e04d1526d03 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -10,6 +10,7 @@ package cloudwatch import ( "context" "errors" + "strings" "testing" "time" @@ -1560,7 +1561,14 @@ func TestInsertTags(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { +<<<<<<< HEAD insertTags(events, c.identifier, resourceTagMap) +======= + subIdentifiers := strings.Split(c.identifier, dimensionSeparator) + for _, subIdentifier := range subIdentifiers { + insertTags(events, c.identifier, subIdentifier, resourceTagMap) + } +>>>>>>> 50c6ae5f3a ([AWS] Add tags to events based on parsed identifier (#33472)) value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) assert.Equal(t, c.expectedTagValue, value) From c0eaa5dc53b9c4e01a8fd1ce10b5d1340e6e9af6 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 1 Nov 2022 09:26:00 -0600 Subject: [PATCH 2/3] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5516c2a60f5..18076ecb54f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -61,14 +61,6 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Metricbeat* -- Fix GCP storage field naming {pull}32806[32806] -- in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305] -- Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] -- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032] -- Change max query size for GetMetricData API to 500 and add RecentlyActive for ListMetrics API call {pull}33105[33105] -- Add GCP CloudSQL region filter {pull}32943[32943] -- Fix logstash cgroup mappings {pull}33131[33131] -- Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263] - Add tags to events based on parsed identifier. {pull}33472[33472] *Packetbeat* From 8cbddbbb7cad24640db7f48a6d138e458c5cae80 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 1 Nov 2022 11:39:43 -0600 Subject: [PATCH 3/3] fix merge conflict --- .../module/aws/cloudwatch/cloudwatch.go | 43 +++---------------- .../module/aws/cloudwatch/cloudwatch_test.go | 4 -- 2 files changed, 5 insertions(+), 42 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 1a3418691bf..77ba174ef3c 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -573,7 +573,6 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient } identifierValue := labels[identifierValueIdx] -<<<<<<< HEAD if _, ok := events[identifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize @@ -585,11 +584,6 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient } events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) - // add tags to event based on identifierValue - insertTags(events, identifierValue, resourceTagMap) -======= - uniqueIdentifierValue := *output.Label + fmt.Sprint("-", valI) - // add tags to event based on identifierValue // Check if identifier includes dimensionSeparator (comma in this case), // split the identifier and check for each sub-identifier. @@ -597,19 +591,17 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient // And tags are only store under s3BucketName in resourceTagMap. subIdentifiers := strings.Split(identifierValue, dimensionSeparator) for _, subIdentifier := range subIdentifiers { - if _, ok := events[uniqueIdentifierValue]; !ok { + if _, ok := events[identifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize // an event for this identifier. if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { continue } - events[uniqueIdentifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) + events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + insertTags(events, identifierValue, subIdentifier, resourceTagMap) } - events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) - insertTags(events, uniqueIdentifierValue, subIdentifier, resourceTagMap) } ->>>>>>> 50c6ae5f3a ([AWS] Add tags to events based on parsed identifier (#33472)) } } } @@ -649,31 +641,7 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } -<<<<<<< HEAD -func insertTags(events map[string]mb.Event, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { - // Check if identifier includes dimensionSeparator (comma in this case), - // split the identifier and check for each sub-identifier. - // For example, identifier might be [storageType, s3BucketName]. - // And tags are only store under s3BucketName in resourceTagMap. - subIdentifiers := strings.Split(identifier, dimensionSeparator) - for _, v := range subIdentifiers { - tags := resourceTagMap[v] - // some metric dimension values are arn format, eg: AWS/DDOS namespace metric - if len(tags) == 0 && strings.HasPrefix(v, "arn:") { - resourceID, err := aws.FindShortIdentifierFromARN(v) - if err == nil { - tags = resourceTagMap[resourceID] - } - } - if len(tags) != 0 { - // By default, replace dot "." using underscore "_" for tag keys. - // Note: tag values are not dedotted. - for _, tag := range tags { - _, _ = events[identifier].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) - } - continue -======= -func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { +func insertTags(events map[string]mb.Event, identifier string, subIdentifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { tags := resourceTagMap[subIdentifier] // some metric dimension values are arn format, eg: AWS/DDOS namespace metric if len(tags) == 0 && strings.HasPrefix(subIdentifier, "arn:") { @@ -686,8 +654,7 @@ func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIde // By default, replace dot "." using underscore "_" for tag keys. // Note: tag values are not dedotted. for _, tag := range tags { - _, _ = events[uniqueIdentifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) ->>>>>>> 50c6ae5f3a ([AWS] Add tags to events based on parsed identifier (#33472)) + _, _ = events[identifier].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) } } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index e04d1526d03..c43a38960a2 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1561,14 +1561,10 @@ func TestInsertTags(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { -<<<<<<< HEAD - insertTags(events, c.identifier, resourceTagMap) -======= subIdentifiers := strings.Split(c.identifier, dimensionSeparator) for _, subIdentifier := range subIdentifiers { insertTags(events, c.identifier, subIdentifier, resourceTagMap) } ->>>>>>> 50c6ae5f3a ([AWS] Add tags to events based on parsed identifier (#33472)) value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) assert.Equal(t, c.expectedTagValue, value)