From 4eda24fb6b1c4db35095a6a0b0ccd831acde3fbf Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 6 Sep 2024 12:51:26 +0300 Subject: [PATCH 01/34] adding fix for aws tags of cloudwatch --- .../module/aws/cloudwatch/cloudwatch.go | 106 ++++++++++-------- .../module/aws/cloudwatch/cloudwatch_test.go | 2 +- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 7b16f6c08b8..402f85105d5 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -156,7 +156,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } - m.logger.Debugf("Collected metrics of metrics = %d", len(eventsWithIdentifier)) + m.logger.Infof("Collected metrics of metrics = %d", len(eventsWithIdentifier)) for _, event := range eventsWithIdentifier { _ = event.RootFields.Delete(aws.CloudWatchPeriodName) @@ -195,7 +195,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // get resource type filters and tags filters for each namespace resourceTypeTagFilters := constructTagsFilters(namespaceDetails) - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) @@ -511,6 +510,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient m.logger.Debugf("resourceType = %s", resourceType) m.logger.Debugf("tagsFilter = %s", tagsFilter) resourceTagMap, err := aws.GetResourcesTags(svcResourceAPI, []string{resourceType}) + if err != nil { // If GetResourcesTags failed, continue report event just without tags. m.logger.Info(fmt.Errorf("getResourcesTags failed, skipping region %s: %w", regionName, err)) @@ -521,65 +521,73 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient } // filter resourceTagMap - for identifier, tags := range resourceTagMap { + for identifierapi, tags := range resourceTagMap { if exists := aws.CheckTagFiltersExist(tagsFilter, tags); !exists { - m.logger.Debugf("In region %s, service %s tags does not match tags_filter", regionName, identifier) - delete(resourceTagMap, identifier) + m.logger.Debugf("In region %s, service %s tags does not match tags_filter", regionName, identifierapi) + delete(resourceTagMap, identifierapi) continue } - m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifier) - } + m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifierapi) + for _, output := range metricDataResults { + if len(output.Values) == 0 { + continue + } - for _, output := range metricDataResults { - if len(output.Values) == 0 { - continue - } + labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator) + for valI, metricDataResultValue := range output.Values { + if len(labels) != aws.LabelConst.LabelLengthTotal { + // if there is no tag in labels but there is a tagsFilter, then no event should be reported. + if len(tagsFilter) != 0 { + continue + } - labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator) - for valI, metricDataResultValue := range output.Values { - if len(labels) != aws.LabelConst.LabelLengthTotal { - // if there is no tag in labels but there is a tagsFilter, then no event should be reported. - if len(tagsFilter) != 0 { + // when there is no identifier value in label, use id+label+region+accountID+namespace+index instead + identifier := labels[aws.LabelConst.AccountIdIdx] + labels[aws.LabelConst.AccountLabelIdx] + regionName + m.MonitoringAccountID + labels[aws.LabelConst.NamespaceIdx] + fmt.Sprint("-", valI) + if _, ok := events[identifier]; !ok { + if labels[aws.LabelConst.AccountIdIdx] != "" { + events[identifier] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) + } else { + events[identifier] = aws.InitEvent(regionName, m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) + } + } + events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } - // when there is no identifier value in label, use id+label+region+accountID+namespace+index instead - identifier := labels[aws.LabelConst.AccountIdIdx] + labels[aws.LabelConst.AccountLabelIdx] + regionName + m.MonitoringAccountID + labels[aws.LabelConst.NamespaceIdx] + fmt.Sprint("-", valI) - if _, ok := events[identifier]; !ok { - if labels[aws.LabelConst.AccountIdIdx] != "" { - events[identifier] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) - } else { - events[identifier] = aws.InitEvent(regionName, m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) + identifierValue := labels[aws.LabelConst.IdentifierValueIdx] + uniqueIdentifierValue := identifierValue + fmt.Sprint("-", valI) + + // add tags to event based on identifierValue + // Check if identifier includes dimensionSeparator (comma in this case), + // split the identifier and check for each sub-identifier. + // For example, identifier might be [storageType, s3BucketName]. + // And tags are only store under s3BucketName in resourceTagMap. + subIdentifiers := strings.Split(identifierValue, dimensionSeparator) + for _, subIdentifier := range subIdentifiers { + if _, ok := events[uniqueIdentifierValue]; !ok { + // when tagsFilter is not empty but no entry in + // resourceTagMap for this identifier, do not initialize + // an event for this identifier. + + // if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { + // continue + // } + events[uniqueIdentifierValue] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) } - } - events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) - continue - } - - identifierValue := labels[aws.LabelConst.IdentifierValueIdx] - uniqueIdentifierValue := identifierValue + fmt.Sprint("-", valI) - - // add tags to event based on identifierValue - // Check if identifier includes dimensionSeparator (comma in this case), - // split the identifier and check for each sub-identifier. - // For example, identifier might be [storageType, s3BucketName]. - // And tags are only store under s3BucketName in resourceTagMap. - subIdentifiers := strings.Split(identifierValue, dimensionSeparator) - for _, subIdentifier := range subIdentifiers { - if _, ok := events[uniqueIdentifierValue]; !ok { - // when tagsFilter is not empty but no entry in - // resourceTagMap for this identifier, do not initialize - // an event for this identifier. - if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { - continue + events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) + m.Logger().Infof("PASSSS 5 insertTags: %v / %v / %v / %v\n", events, uniqueIdentifierValue, subIdentifier, resourceTagMap) + insertTags(events, uniqueIdentifierValue, subIdentifier, identifierapi, resourceTagMap) + tags := resourceTagMap[subIdentifier] + for _, tag := range tags { + m.Logger().Infof("PASSSS 6 Print: %s , %s \n", *tag.Key, *tag.Value) } - events[uniqueIdentifierValue] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) + m.Logger().Infof("PASSSS 6 insertTags: %v / %v / %v / %v\n", events, uniqueIdentifierValue, subIdentifier, identifierapi) + } - events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) - insertTags(events, uniqueIdentifierValue, subIdentifier, resourceTagMap) } } } + } return events, nil } @@ -617,8 +625,8 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } -func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { - tags := resourceTagMap[subIdentifier] +func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, identifierapi string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { + tags := resourceTagMap[identifierapi] // some metric dimension values are arn format, eg: AWS/DDOS namespace metric if len(tags) == 0 && strings.HasPrefix(subIdentifier, "arn:") { resourceID, err := aws.FindShortIdentifierFromARN(subIdentifier) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 3c355d24612..3378bebe58a 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1478,7 +1478,7 @@ func TestInsertTags(t *testing.T) { t.Run(c.title, func(t *testing.T) { subIdentifiers := strings.Split(c.identifier, dimensionSeparator) for _, subIdentifier := range subIdentifiers { - insertTags(events, c.identifier, subIdentifier, resourceTagMap) + insertTags(events, c.identifier, subIdentifier, subIdentifier, resourceTagMap) } value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) From 1750c5479d288c7e4fbe210ff493c6bfa04ad454 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Tue, 10 Sep 2024 19:07:30 +0300 Subject: [PATCH 02/34] working for the first aws fix --- x-pack/metricbeat/module/aws/aws.go | 1 + .../module/aws/cloudwatch/cloudwatch.go | 141 ++++++++++-------- .../module/aws/cloudwatch/cloudwatch_test.go | 22 ++- x-pack/metricbeat/module/aws/utils.go | 55 +++++++ 4 files changed, 151 insertions(+), 68 deletions(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 90a9e0d416d..f920be51a3d 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -36,6 +36,7 @@ type Config struct { AWSConfig awscommon.ConfigAWS `config:",inline"` TagsFilter []Tag `config:"tags_filter"` IncludeLinkedAccounts *bool `config:"include_linked_accounts"` + LimitRestAPI int64 `config:"limit_rest_api"` } // MetricSet is the base metricset for all aws metricsets diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 402f85105d5..5780b02cc5f 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -12,6 +12,7 @@ import ( "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" @@ -121,6 +122,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency) m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) + // Initialise the map that will be used in case APiGW api is configured + infoapi := make(map[string]string) // Check statistic method in config err := m.checkStatistics() @@ -146,17 +149,17 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - svcCloudwatch, svcResourceAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + svcCloudwatch, svcResourceAPI, svcRestAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName) } - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime) + eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, svcRestAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } - m.logger.Infof("Collected metrics of metrics = %d", len(eventsWithIdentifier)) + m.logger.Debugf("Collected metrics of metrics = %d", len(eventsWithIdentifier)) for _, event := range eventsWithIdentifier { _ = event.RootFields.Delete(aws.CloudWatchPeriodName) @@ -171,7 +174,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - svcCloudwatch, svcResourceAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + svcCloudwatch, svcResourceAPI, svcRestAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName, err) continue @@ -195,7 +198,17 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // get resource type filters and tags filters for each namespace resourceTypeTagFilters := constructTagsFilters(namespaceDetails) - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + + //Check whether namespace is APIGW + checkns := "AWS/ApiGateway" + if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { + infoapi, err = aws.GetRestAPIsOutput(svcRestAPI) + if err != nil { + m.Logger().Errorf("could not get rest apis output: %v", err) + } + } + + eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, svcRestAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } @@ -218,7 +231,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // createAwsRequiredClients will return the two necessary client instances to do Metric requests to the AWS API -func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, error) { +func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, *apigateway.Client, error) { m.logger.Debugf("Collecting metrics from AWS region %s", regionName) svcCloudwatchClient := cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) { @@ -234,7 +247,11 @@ func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionNa } }) - return svcCloudwatchClient, svcResourceAPIClient, nil + svcGetRestAPIClient := apigateway.NewFromConfig(beatsConfig, func(o *apigateway.Options) { + + }) + + return svcCloudwatchClient, svcResourceAPIClient, svcGetRestAPIClient, nil } // filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match @@ -455,7 +472,7 @@ func insertRootFields(event mb.Event, metricValue float64, labels []string) mb.E return event } -func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) { +func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, svcRestAPI apigateway.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, infoAPImap map[string]string, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) { // Initialize events for each identifier. events := make(map[string]mb.Event) @@ -510,7 +527,6 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient m.logger.Debugf("resourceType = %s", resourceType) m.logger.Debugf("tagsFilter = %s", tagsFilter) resourceTagMap, err := aws.GetResourcesTags(svcResourceAPI, []string{resourceType}) - if err != nil { // If GetResourcesTags failed, continue report event just without tags. m.logger.Info(fmt.Errorf("getResourcesTags failed, skipping region %s: %w", regionName, err)) @@ -521,73 +537,70 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient } // filter resourceTagMap - for identifierapi, tags := range resourceTagMap { + for identifier, tags := range resourceTagMap { if exists := aws.CheckTagFiltersExist(tagsFilter, tags); !exists { - m.logger.Debugf("In region %s, service %s tags does not match tags_filter", regionName, identifierapi) - delete(resourceTagMap, identifierapi) + m.logger.Debugf("In region %s, service %s tags does not match tags_filter", regionName, identifier) + delete(resourceTagMap, identifier) continue } - m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifierapi) - for _, output := range metricDataResults { - if len(output.Values) == 0 { - continue - } + m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifier) + } - labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator) - for valI, metricDataResultValue := range output.Values { - if len(labels) != aws.LabelConst.LabelLengthTotal { - // if there is no tag in labels but there is a tagsFilter, then no event should be reported. - if len(tagsFilter) != 0 { - continue - } + for _, output := range metricDataResults { + if len(output.Values) == 0 { + continue + } - // when there is no identifier value in label, use id+label+region+accountID+namespace+index instead - identifier := labels[aws.LabelConst.AccountIdIdx] + labels[aws.LabelConst.AccountLabelIdx] + regionName + m.MonitoringAccountID + labels[aws.LabelConst.NamespaceIdx] + fmt.Sprint("-", valI) - if _, ok := events[identifier]; !ok { - if labels[aws.LabelConst.AccountIdIdx] != "" { - events[identifier] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) - } else { - events[identifier] = aws.InitEvent(regionName, m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) - } - } - events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) + labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator) + for valI, metricDataResultValue := range output.Values { + if len(labels) != aws.LabelConst.LabelLengthTotal { + // if there is no tag in labels but there is a tagsFilter, then no event should be reported. + if len(tagsFilter) != 0 { continue } - identifierValue := labels[aws.LabelConst.IdentifierValueIdx] - uniqueIdentifierValue := identifierValue + fmt.Sprint("-", valI) - - // add tags to event based on identifierValue - // Check if identifier includes dimensionSeparator (comma in this case), - // split the identifier and check for each sub-identifier. - // For example, identifier might be [storageType, s3BucketName]. - // And tags are only store under s3BucketName in resourceTagMap. - subIdentifiers := strings.Split(identifierValue, dimensionSeparator) - for _, subIdentifier := range subIdentifiers { - if _, ok := events[uniqueIdentifierValue]; !ok { - // when tagsFilter is not empty but no entry in - // resourceTagMap for this identifier, do not initialize - // an event for this identifier. - - // if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { - // continue - // } - events[uniqueIdentifierValue] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) - } - events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) - m.Logger().Infof("PASSSS 5 insertTags: %v / %v / %v / %v\n", events, uniqueIdentifierValue, subIdentifier, resourceTagMap) - insertTags(events, uniqueIdentifierValue, subIdentifier, identifierapi, resourceTagMap) - tags := resourceTagMap[subIdentifier] - for _, tag := range tags { - m.Logger().Infof("PASSSS 6 Print: %s , %s \n", *tag.Key, *tag.Value) + // when there is no identifier value in label, use id+label+region+accountID+namespace+index instead + identifier := labels[aws.LabelConst.AccountIdIdx] + labels[aws.LabelConst.AccountLabelIdx] + regionName + m.MonitoringAccountID + labels[aws.LabelConst.NamespaceIdx] + fmt.Sprint("-", valI) + if _, ok := events[identifier]; !ok { + if labels[aws.LabelConst.AccountIdIdx] != "" { + events[identifier] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) + } else { + events[identifier] = aws.InitEvent(regionName, m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) } - m.Logger().Infof("PASSSS 6 insertTags: %v / %v / %v / %v\n", events, uniqueIdentifierValue, subIdentifier, identifierapi) + } + events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) + continue + } + identifierValue := labels[aws.LabelConst.IdentifierValueIdx] + uniqueIdentifierValue := identifierValue + fmt.Sprint("-", valI) + + // add tags to event based on identifierValue + // Check if identifier includes dimensionSeparator (comma in this case), + // split the identifier and check for each sub-identifier. + // For example, identifier might be [storageType, s3BucketName]. + // And tags are only store under s3BucketName in resourceTagMap. + subIdentifiers := strings.Split(identifierValue, dimensionSeparator) + for _, subIdentifier := range subIdentifiers { + for apiId, apiName := range infoAPImap { + if apiName == subIdentifier { + subIdentifier = apiId + } } + if _, ok := events[uniqueIdentifierValue]; !ok { + // when tagsFilter is not empty but no entry in + // resourceTagMap for this identifier, do not initialize + // an event for this identifier. + if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { + continue + } + events[uniqueIdentifierValue] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI], labels[aws.LabelConst.PeriodLabelIdx]) + } + events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) + insertTags(events, uniqueIdentifierValue, subIdentifier, resourceTagMap) } } } - } return events, nil } @@ -625,8 +638,8 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } -func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, identifierapi string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { - tags := resourceTagMap[identifierapi] +func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, subIdentifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { + tags := resourceTagMap[subIdentifier] // some metric dimension values are arn format, eg: AWS/DDOS namespace metric if len(tags) == 0 && strings.HasPrefix(subIdentifier, "arn:") { resourceID, err := aws.FindShortIdentifierFromARN(subIdentifier) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 3378bebe58a..bf9aac73198 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/service/apigateway" + restapi "github.com/aws/aws-sdk-go-v2/service/apigateway" cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" resourcegroupstaggingapitypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" "github.com/aws/smithy-go/middleware" @@ -1255,6 +1257,14 @@ func (m *MockResourceGroupsTaggingClient) GetResources(context.Context, *resourc }, nil } +// MockResourceGroupsTaggingClient2 is used for unit tests. +type MockResourceGroupsTaggingClient2 struct{} + +// GetResources implements resourcegroupstaggingapi.GetResourcesAPIClient. +func (m *MockResourceGroupsTaggingClient2) GetResources(context.Context, *resourcegroupstaggingapi.GetResourcesInput, ...func(*apigateway.Options)) (*restapi.GetResourcesOutput, error) { + return &restapi.GetResourcesOutput{}, nil +} + func TestCreateEventsWithIdentifier(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} @@ -1262,6 +1272,9 @@ func TestCreateEventsWithIdentifier(t *testing.T) { m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} + mockTaggingSvc2 := &MockResourceGroupsTaggingClient2{} + infoAPImap := make(map[string]string) + mockCloudwatchSvc := &MockCloudWatchClient{} listMetricWithStatsTotal := []metricsWithStatistics{{ listMetric1, @@ -1270,8 +1283,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { resourceTypeTagFilters := map[string][]aws.Tag{} resourceTypeTagFilters["ec2:instance"] = nameTestEC2Tag startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) - - events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, mockTaggingSvc2, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) assert.Equal(t, 1, len(events)) @@ -1292,6 +1304,8 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { mockTaggingSvc := &MockResourceGroupsTaggingClient{} mockCloudwatchSvc := &MockCloudWatchClientWithoutDim{} + mockTaggingSvc2 := &MockResourceGroupsTaggingClient2{} + infoAPImap := make(map[string]string) listMetricWithStatsTotal := []metricsWithStatistics{ { cloudwatchMetric: aws.MetricWithID{ @@ -1316,7 +1330,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { resourceTypeTagFilters := map[string][]aws.Tag{} startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) - events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, mockTaggingSvc2, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) expectedID := " " + " " + regionName + accountID + namespace @@ -1478,7 +1492,7 @@ func TestInsertTags(t *testing.T) { t.Run(c.title, func(t *testing.T) { subIdentifiers := strings.Split(c.identifier, dimensionSeparator) for _, subIdentifier := range subIdentifiers { - insertTags(events, c.identifier, subIdentifier, subIdentifier, resourceTagMap) + insertTags(events, c.identifier, subIdentifier, resourceTagMap) } value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 88168a33f74..ad72a30d1cb 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -12,10 +12,12 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" resourcegroupstaggingapitypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" + "github.com/aws/aws-sdk-go/aws" ) const DefaultApiTimeout = 5 * time.Second @@ -92,6 +94,25 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat return metricWithAccountID, nil } +// GetRestAPIsOutput function gets results from apigw api. +func GetRestAPIsOutput(svcRestApi *apigateway.Client) (map[string]string, error) { + input := &apigateway.GetRestApisInput{} + ctx, cancel := getContextWithTimeout(DefaultApiTimeout) + defer cancel() + result, err := svcRestApi.GetRestApis(ctx, input) + if err != nil { + return nil, fmt.Errorf("error GetRestApis %w", err) + } + + // Iterate and display the APIs + infoAPImap := make(map[string]string, len(result.Items)) + for _, api := range result.Items { + + infoAPImap[aws.StringValue(api.Name)] = aws.StringValue(api.Id) + } + return infoAPImap, nil +} + // GetMetricDataResults function uses MetricDataQueries to get metric data output. func GetMetricDataResults(metricDataQueries []types.MetricDataQuery, svc cloudwatch.GetMetricDataAPIClient, startTime time.Time, endTime time.Time) ([]types.MetricDataResult, error) { maxNumberOfMetricsRetrieved := 500 @@ -126,6 +147,40 @@ func GetMetricDataResults(metricDataQueries []types.MetricDataQuery, svc cloudwa return getMetricDataOutput.MetricDataResults, nil } +// Get function uses apigateway get-rest-api to get data output. +func GetRestApiResults(metricDataQueries []types.MetricDataQuery, svc cloudwatch.GetMetricDataAPIClient, startTime time.Time, endTime time.Time) ([]types.MetricDataResult, error) { + maxNumberOfMetricsRetrieved := 500 + getMetricDataOutput := &cloudwatch.GetMetricDataOutput{NextToken: nil} + + // Split metricDataQueries into smaller slices that length no longer than 500. + // 500 is defined in maxNumberOfMetricsRetrieved. + // To avoid ValidationError: The collection MetricDataQueries must not have a size greater than 500. + for i := 0; i < len(metricDataQueries); i += maxNumberOfMetricsRetrieved { + metricDataQueriesPartial := metricDataQueries[i:int(math.Min(float64(i+maxNumberOfMetricsRetrieved), float64(len(metricDataQueries))))] + if len(metricDataQueriesPartial) == 0 { + return getMetricDataOutput.MetricDataResults, nil + } + + getMetricDataInput := &cloudwatch.GetMetricDataInput{ + StartTime: &startTime, + EndTime: &endTime, + MetricDataQueries: metricDataQueriesPartial, + } + + paginator := cloudwatch.NewGetMetricDataPaginator(svc, getMetricDataInput) + var err error + var page *cloudwatch.GetMetricDataOutput + for paginator.HasMorePages() { + if page, err = paginator.NextPage(context.TODO()); err != nil { + return getMetricDataOutput.MetricDataResults, fmt.Errorf("error GetMetricData with Paginator: %w", err) + } + getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, page.MetricDataResults...) + } + } + + return getMetricDataOutput.MetricDataResults, nil +} + // CheckTimestampInArray checks if input timestamp exists in timestampArray and if it exists, return the position. func CheckTimestampInArray(timestamp time.Time, timestampArray []time.Time) (bool, int) { for i := 0; i < len(timestampArray); i++ { From cf7db5d4ea5ad13bf202375c152d24fe8150f84a Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 11 Sep 2024 17:28:44 +0300 Subject: [PATCH 03/34] first working sample for aws tags with filters --- go.mod | 9 +++---- go.sum | 14 ++++++----- .../module/aws/cloudwatch/cloudwatch.go | 24 +++++++++++-------- .../module/aws/cloudwatch/cloudwatch_test.go | 19 ++++++++------- x-pack/metricbeat/module/aws/utils.go | 1 + 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index a47bf7ec154..ab93e14d093 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/aws/aws-lambda-go v1.44.0 - github.com/aws/aws-sdk-go-v2 v1.30.4 + github.com/aws/aws-sdk-go-v2 v1.30.5 github.com/aws/aws-sdk-go-v2/config v1.27.29 github.com/aws/aws-sdk-go-v2/credentials v1.17.29 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5 @@ -182,8 +182,10 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/go-autorest/autorest/adal v0.9.24 github.com/apache/arrow/go/v14 v14.0.2 + github.com/aws/aws-sdk-go v1.38.60 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13 + github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5 github.com/aws/aws-sdk-go-v2/service/health v1.26.4 github.com/aws/aws-sdk-go-v2/service/kinesis v1.29.5 @@ -248,10 +250,9 @@ require ( github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/apache/thrift v0.19.0 // indirect github.com/armon/go-radix v1.0.0 // indirect - github.com/aws/aws-sdk-go v1.38.60 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect diff --git a/go.sum b/go.sum index ec32ee823dd..3b209b9e3d1 100644 --- a/go.sum +++ b/go.sum @@ -298,8 +298,8 @@ github.com/aws/aws-sdk-go v1.38.60/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= -github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= @@ -316,17 +316,19 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33/go.mod h1:84XgODVR8uRhm github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13 h1:X8EeaOjl91c8sP14NG8EHx5ZxXLJg0tHDp+KQSghp28= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13/go.mod h1:kEI/h2bETfm09LSd7xEEH2qcU1cd//+5HH4Le7p9JgY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= +github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8 h1:CgEyY7gfTf7lHYcCi7+w6jJ1XQBugjpadtsuN3TGxdQ= +github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8/go.mod h1:z99ur4Ha5540t8hb5XtqV/UMOnEoEZK22lhr5ZBS0zw= github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5 h1:YeTVIy7cJLeahs7K0jQGDGAd1YYND/to/z8N3kqZBhY= github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5/go.mod h1:y45SdA9v+dLlweaqwAQMoFeXqdRvgwevafa2X8iTqZQ= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5 h1:/YvqO1j75i4leoV+Z3a5s/dAlEszf2wTKBW8jc3Gd4s= diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 5780b02cc5f..31349d5a66e 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -149,12 +149,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - svcCloudwatch, svcResourceAPI, svcRestAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + svcCloudwatch, svcResourceAPI, _, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName) } - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, svcRestAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime) + eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } @@ -207,8 +207,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.Logger().Errorf("could not get rest apis output: %v", err) } } - - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, svcRestAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) + m.Logger().Infof("PASSS InfoAPI response: %v", infoapi) + eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } @@ -472,7 +472,7 @@ func insertRootFields(event mb.Event, metricValue float64, labels []string) mb.E return event } -func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, svcRestAPI apigateway.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, infoAPImap map[string]string, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) { +func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, infoAPImap map[string]string, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) { // Initialize events for each identifier. events := make(map[string]mb.Event) @@ -543,7 +543,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient delete(resourceTagMap, identifier) continue } - m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifier) + m.logger.Infof("In region %s, service %s tags match tags_filter", regionName, identifier) } for _, output := range metricDataResults { @@ -582,11 +582,15 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient // And tags are only store under s3BucketName in resourceTagMap. subIdentifiers := strings.Split(identifierValue, dimensionSeparator) for _, subIdentifier := range subIdentifiers { - for apiId, apiName := range infoAPImap { - if apiName == subIdentifier { - subIdentifier = apiId - } + m.logger.Infof("PASSSS Before %s", subIdentifier) + + if valAPIName, ok := infoAPImap[subIdentifier]; ok { + subIdentifier = valAPIName + m.logger.Infof("PASSSS After %s from api %s", subIdentifier, valAPIName) } + + m.logger.Infof("PASSSS ola mazi %s", resourceTagMap[subIdentifier]) + if _, ok := events[uniqueIdentifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index bf9aac73198..f718ce19046 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1261,7 +1261,7 @@ func (m *MockResourceGroupsTaggingClient) GetResources(context.Context, *resourc type MockResourceGroupsTaggingClient2 struct{} // GetResources implements resourcegroupstaggingapi.GetResourcesAPIClient. -func (m *MockResourceGroupsTaggingClient2) GetResources(context.Context, *resourcegroupstaggingapi.GetResourcesInput, ...func(*apigateway.Options)) (*restapi.GetResourcesOutput, error) { +func (m *MockResourceGroupsTaggingClient2) GetResources(context.Context, *apigateway.GetResourcesInput, ...func(*apigateway.Options)) (*restapi.GetResourcesOutput, error) { return &restapi.GetResourcesOutput{}, nil } @@ -1272,7 +1272,6 @@ func TestCreateEventsWithIdentifier(t *testing.T) { m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} - mockTaggingSvc2 := &MockResourceGroupsTaggingClient2{} infoAPImap := make(map[string]string) mockCloudwatchSvc := &MockCloudWatchClient{} @@ -1283,7 +1282,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { resourceTypeTagFilters := map[string][]aws.Tag{} resourceTypeTagFilters["ec2:instance"] = nameTestEC2Tag startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) - events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, mockTaggingSvc2, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) assert.Equal(t, 1, len(events)) @@ -1304,7 +1303,6 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { mockTaggingSvc := &MockResourceGroupsTaggingClient{} mockCloudwatchSvc := &MockCloudWatchClientWithoutDim{} - mockTaggingSvc2 := &MockResourceGroupsTaggingClient2{} infoAPImap := make(map[string]string) listMetricWithStatsTotal := []metricsWithStatistics{ { @@ -1330,7 +1328,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { resourceTypeTagFilters := map[string][]aws.Tag{} startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) - events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, mockTaggingSvc2, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) expectedID := " " + " " + regionName + accountID + namespace @@ -1351,6 +1349,7 @@ func TestCreateEventsWithDataGranularity(t *testing.T) { mockTaggingSvc := &MockResourceGroupsTaggingClient{} mockCloudwatchSvc := &MockCloudWatchClientWithDataGranularity{} + infoAPImap := make(map[string]string) listMetricWithStatsTotal := []metricsWithStatistics{ { listMetric1, @@ -1365,7 +1364,7 @@ func TestCreateEventsWithDataGranularity(t *testing.T) { resourceTypeTagFilters := map[string][]aws.Tag{} startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) - events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) expectedID := " " + regionName + accountID @@ -1391,6 +1390,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} + infoAPImap := make(map[string]string) mockCloudwatchSvc := &MockCloudWatchClient{} listMetricWithStatsTotal := []metricsWithStatistics{ { @@ -1408,7 +1408,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { resourceTypeTagFilters["ec2:instance"] = nameTestEC2Tag startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) - events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) assert.Equal(t, 1, len(events)) @@ -1420,7 +1420,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { }, } - events, err = m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + events, err = m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) assert.Equal(t, 0, len(events)) } @@ -1570,11 +1570,12 @@ func TestCreateEventsTimestamp(t *testing.T) { } resourceTypeTagFilters := map[string][]aws.Tag{} + infoAPImap := make(map[string]string) startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) cloudwatchMock := &MockCloudWatchClientWithoutDim{} resGroupTaggingClientMock := &MockResourceGroupsTaggingClient{} - events, err := m.createEvents(cloudwatchMock, resGroupTaggingClientMock, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + events, err := m.createEvents(cloudwatchMock, resGroupTaggingClientMock, listMetricWithStatsTotal, resourceTypeTagFilters, infoAPImap, regionName, startTime, endTime) assert.NoError(t, err) assert.Equal(t, timestamp, events[" "+regionName+accountID+namespace+"-0"].Timestamp) } diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index ad72a30d1cb..a6f7c6e7f18 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -95,6 +95,7 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat } // GetRestAPIsOutput function gets results from apigw api. +// GetRestApis Apigateway API is used to retrieve the specified info. This returns a map with the names and ids of RestAPis configured func GetRestAPIsOutput(svcRestApi *apigateway.Client) (map[string]string, error) { input := &apigateway.GetRestApisInput{} ctx, cancel := getContextWithTimeout(DefaultApiTimeout) From 64de6bfe20e7000cea4f70b1ebca9dfeff6d4793 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 10:50:14 +0300 Subject: [PATCH 04/34] adding CHANGELOG.next --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0ece7c88964..d9a2f5cc980 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -326,6 +326,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add new metricset network for the vSphere module. {pull}40559[40559] - Add new metricset resourcepool for the vSphere module. {pull}40456[40456] - Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661] +- Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway eresources {pull}40755[40755] *Metricbeat* From 7db4e0e05971f966f94d8f46745f2213f20fb590 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 10:53:43 +0300 Subject: [PATCH 05/34] adding CHANGELOG.next --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 61f5437bd46..33579a3c590 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -329,7 +329,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add new metricset network for the vSphere module. {pull}40559[40559] - Add new metricset resourcepool for the vSphere module. {pull}40456[40456] - Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661] -- Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway eresources {pull}40755[40755] +- Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway resources {pull}40755[40755] - Add new metricset datastorecluster for vSphere module. {pull}40634[40634] {pull}40694[40694] - Add new metrics for the vSphere Virtualmachine metricset. {pull}40485[40485] - Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683] From f4fd00f8a56fd6a4a76f3da85892561dbf275f8c Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 10:54:30 +0300 Subject: [PATCH 06/34] adding CHANGELOG.next --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 33579a3c590..a81f6ac48e3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -328,7 +328,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add new metricset cluster for the vSphere module. {pull}40536[40536] - Add new metricset network for the vSphere module. {pull}40559[40559] - Add new metricset resourcepool for the vSphere module. {pull}40456[40456] -- Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661] - Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway resources {pull}40755[40755] - Add new metricset datastorecluster for vSphere module. {pull}40634[40634] {pull}40694[40694] - Add new metrics for the vSphere Virtualmachine metricset. {pull}40485[40485] From dc5e38b80ce49898b1b6791fc695e75835eb185e Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 11:25:45 +0300 Subject: [PATCH 07/34] minor changes --- x-pack/metricbeat/module/aws/aws.go | 2 +- .../module/aws/cloudwatch/cloudwatch.go | 2 +- x-pack/metricbeat/module/aws/utils.go | 47 ++++--------------- 3 files changed, 11 insertions(+), 40 deletions(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f920be51a3d..bcd0ef79705 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -36,7 +36,7 @@ type Config struct { AWSConfig awscommon.ConfigAWS `config:",inline"` TagsFilter []Tag `config:"tags_filter"` IncludeLinkedAccounts *bool `config:"include_linked_accounts"` - LimitRestAPI int64 `config:"limit_rest_api"` + LimitRestAPI *int32 `config:"limit_rest_api"` } // MetricSet is the base metricset for all aws metricsets diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 31349d5a66e..1024ccf9d31 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -202,7 +202,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { //Check whether namespace is APIGW checkns := "AWS/ApiGateway" if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { - infoapi, err = aws.GetRestAPIsOutput(svcRestAPI) + infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index a6f7c6e7f18..902a1854f90 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -95,20 +95,25 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat } // GetRestAPIsOutput function gets results from apigw api. -// GetRestApis Apigateway API is used to retrieve the specified info. This returns a map with the names and ids of RestAPis configured -func GetRestAPIsOutput(svcRestApi *apigateway.Client) (map[string]string, error) { +// GetRestApis Apigateway API is used to retrieve the specified info. This returns a map with the names and ids of RestAPIs configured +// Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. +func GetRestAPIsOutput(svcRestApi *apigateway.Client, limit *int32) (map[string]string, error) { input := &apigateway.GetRestApisInput{} + if limit != nil { + input = &apigateway.GetRestApisInput{ + Limit: limit, + } + } ctx, cancel := getContextWithTimeout(DefaultApiTimeout) defer cancel() result, err := svcRestApi.GetRestApis(ctx, input) if err != nil { - return nil, fmt.Errorf("error GetRestApis %w", err) + return nil, fmt.Errorf("error retrieving GetRestApis %w", err) } // Iterate and display the APIs infoAPImap := make(map[string]string, len(result.Items)) for _, api := range result.Items { - infoAPImap[aws.StringValue(api.Name)] = aws.StringValue(api.Id) } return infoAPImap, nil @@ -148,40 +153,6 @@ func GetMetricDataResults(metricDataQueries []types.MetricDataQuery, svc cloudwa return getMetricDataOutput.MetricDataResults, nil } -// Get function uses apigateway get-rest-api to get data output. -func GetRestApiResults(metricDataQueries []types.MetricDataQuery, svc cloudwatch.GetMetricDataAPIClient, startTime time.Time, endTime time.Time) ([]types.MetricDataResult, error) { - maxNumberOfMetricsRetrieved := 500 - getMetricDataOutput := &cloudwatch.GetMetricDataOutput{NextToken: nil} - - // Split metricDataQueries into smaller slices that length no longer than 500. - // 500 is defined in maxNumberOfMetricsRetrieved. - // To avoid ValidationError: The collection MetricDataQueries must not have a size greater than 500. - for i := 0; i < len(metricDataQueries); i += maxNumberOfMetricsRetrieved { - metricDataQueriesPartial := metricDataQueries[i:int(math.Min(float64(i+maxNumberOfMetricsRetrieved), float64(len(metricDataQueries))))] - if len(metricDataQueriesPartial) == 0 { - return getMetricDataOutput.MetricDataResults, nil - } - - getMetricDataInput := &cloudwatch.GetMetricDataInput{ - StartTime: &startTime, - EndTime: &endTime, - MetricDataQueries: metricDataQueriesPartial, - } - - paginator := cloudwatch.NewGetMetricDataPaginator(svc, getMetricDataInput) - var err error - var page *cloudwatch.GetMetricDataOutput - for paginator.HasMorePages() { - if page, err = paginator.NextPage(context.TODO()); err != nil { - return getMetricDataOutput.MetricDataResults, fmt.Errorf("error GetMetricData with Paginator: %w", err) - } - getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, page.MetricDataResults...) - } - } - - return getMetricDataOutput.MetricDataResults, nil -} - // CheckTimestampInArray checks if input timestamp exists in timestampArray and if it exists, return the position. func CheckTimestampInArray(timestamp time.Time, timestampArray []time.Time) (bool, int) { for i := 0; i < len(timestampArray); i++ { From 1479e8bc795afd0249eb004197cde06ba66b2e0b Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 12:37:40 +0300 Subject: [PATCH 08/34] adding notice --- NOTICE.txt | 672 ++++++++++++------ auditbeat/docs/modules/auditd.asciidoc | 327 --------- .../docs/modules/file_integrity.asciidoc | 181 ----- x-pack/auditbeat/docs/modules/system.asciidoc | 211 ------ .../docs/modules/system/host.asciidoc | 21 - .../docs/modules/system/login.asciidoc | 21 - .../docs/modules/system/package.asciidoc | 21 - .../docs/modules/system/process.asciidoc | 21 - .../docs/modules/system/socket.asciidoc | 21 - .../docs/modules/system/user.asciidoc | 21 - 10 files changed, 442 insertions(+), 1075 deletions(-) delete mode 100644 auditbeat/docs/modules/auditd.asciidoc delete mode 100644 auditbeat/docs/modules/file_integrity.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system/host.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system/login.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system/package.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system/process.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system/socket.asciidoc delete mode 100644 x-pack/auditbeat/docs/modules/system/user.asciidoc diff --git a/NOTICE.txt b/NOTICE.txt index b0a0438eb29..06c4df765d2 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5073,13 +5073,437 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-lambda-go@v1.44 +-------------------------------------------------------------------------------- +Dependency : github.com/aws/aws-sdk-go +Version: v1.38.60 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go@v1.38.60/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2 -Version: v1.30.4 +Version: v1.30.5 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2@v1.30.5/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/aws/aws-sdk-go-v2/config +Version: v1.27.29 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2@v1.30.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/config@v1.27.29/LICENSE.txt: Apache License @@ -5286,12 +5710,12 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2@v1.30 -------------------------------------------------------------------------------- -Dependency : github.com/aws/aws-sdk-go-v2/config -Version: v1.27.29 +Dependency : github.com/aws/aws-sdk-go-v2/credentials +Version: v1.17.29 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/config@v1.27.29/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/credentials@v1.17.29/LICENSE.txt: Apache License @@ -5498,12 +5922,12 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/confi -------------------------------------------------------------------------------- -Dependency : github.com/aws/aws-sdk-go-v2/credentials -Version: v1.17.29 +Dependency : github.com/aws/aws-sdk-go-v2/feature/ec2/imds +Version: v1.16.12 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/credentials@v1.17.29/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/ec2/imds@v1.16.12/LICENSE.txt: Apache License @@ -5710,12 +6134,12 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/crede -------------------------------------------------------------------------------- -Dependency : github.com/aws/aws-sdk-go-v2/feature/ec2/imds -Version: v1.16.12 +Dependency : github.com/aws/aws-sdk-go-v2/feature/s3/manager +Version: v1.17.13 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/ec2/imds@v1.16.12/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/s3/manager@v1.17.13/LICENSE.txt: Apache License @@ -5922,12 +6346,12 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/featu -------------------------------------------------------------------------------- -Dependency : github.com/aws/aws-sdk-go-v2/feature/s3/manager -Version: v1.17.13 +Dependency : github.com/aws/aws-sdk-go-v2/service/apigateway +Version: v1.25.8 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/s3/manager@v1.17.13/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/apigateway@v1.25.8/LICENSE.txt: Apache License @@ -33805,218 +34229,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/aws/aws-sdk-go -Version: v1.38.60 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go@v1.38.60/LICENSE.txt: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream Version: v1.6.4 @@ -34231,11 +34443,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/aws/p -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/internal/configsources -Version: v1.3.16 +Version: v1.3.17 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/configsources@v1.3.16/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/configsources@v1.3.17/LICENSE.txt: Apache License @@ -34443,11 +34655,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/inter -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 -Version: v2.6.16 +Version: v2.6.17 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2@v2.6.16/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2@v2.6.17/LICENSE.txt: Apache License diff --git a/auditbeat/docs/modules/auditd.asciidoc b/auditbeat/docs/modules/auditd.asciidoc deleted file mode 100644 index 0361dc56097..00000000000 --- a/auditbeat/docs/modules/auditd.asciidoc +++ /dev/null @@ -1,327 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -:modulename: auditd - -[id="{beatname_lc}-module-auditd"] -== Auditd Module - -The `auditd` module receives audit events from the Linux Audit Framework that -is a part of the Linux kernel. - -This module is available only for Linux. - -[float] -=== How it works - -This module establishes a subscription to the kernel to receive the events -as they occur. So unlike most other modules, the `period` configuration -option is unused because it is not implemented using polling. - -The Linux Audit Framework can send multiple messages for a single auditable -event. For example, a `rename` syscall causes the kernel to send eight separate -messages. Each message describes a different aspect of the activity that is -occurring (the syscall itself, file paths, current working directory, process -title). This module will combine all of the data from each of the messages -into a single event. - -Messages for one event can be interleaved with messages from another event. This -module will buffer the messages in order to combine related messages into a -single event even if they arrive interleaved or out of order. - -[float] -=== Useful commands - -When running {beatname_uc} with the `auditd` module enabled, you might find -that other monitoring tools interfere with {beatname_uc}. - -For example, you might encounter errors if another process, such as `auditd`, is -registered to receive data from the Linux Audit Framework. You can use these -commands to see if the `auditd` service is running and stop it: - -* See if `auditd` is running: -+ -[source,shell] ------ -service auditd status ------ - -* Stop the `auditd` service: -+ -[source,shell] ------ -service auditd stop ------ - -* Disable `auditd` from starting on boot: -+ -[source,shell] ------ -chkconfig auditd off ------ - -To save CPU usage and disk space, you can use this command to stop `journald` -from listening to audit messages: - -[source,shell] ------ -systemctl mask systemd-journald-audit.socket ------ - -[float] -=== Inspect the kernel audit system status - -{beatname_uc} provides useful commands to query the state of the audit system -in the Linux kernel. - -* See the list of installed audit rules: -+ -[source,shell] ------ -auditbeat show auditd-rules ------ -+ -Prints the list of loaded rules, similar to `auditctl -l`: -+ -[source,shell] ------ --a never,exit -S all -F pid=26253 --a always,exit -F arch=b32 -S all -F key=32bit-abi --a always,exit -F arch=b64 -S execve,execveat -F key=exec --a always,exit -F arch=b64 -S connect,accept,bind -F key=external-access --w /etc/group -p wa -k identity --w /etc/passwd -p wa -k identity --w /etc/gshadow -p wa -k identity --a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F key=access --a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F key=access ------ - -* See the status of the audit system: -+ -[source,shell] ------ -auditbeat show auditd-status ------ -+ -Prints the status of the kernel audit system, similar to `auditctl -s`: -+ -[source,shell] ------ -enabled 1 -failure 0 -pid 0 -rate_limit 0 -backlog_limit 8192 -lost 14407 -backlog 0 -backlog_wait_time 0 -features 0xf ------ - -[float] -=== Configuration options - -This module has some configuration options for tuning its behavior. The -following example shows all configuration options with their default values. - -[source,yaml] ----- -- module: auditd - resolve_ids: true - failure_mode: silent - backlog_limit: 8192 - rate_limit: 0 - include_raw_message: false - include_warnings: false - backpressure_strategy: auto - immutable: false ----- - -This module also supports the -<> -described later. - -*`socket_type`*:: This optional setting controls the type of -socket that {beatname_uc} uses to receive events from the kernel. The two -options are `unicast` and `multicast`. -+ -`unicast` should be used when {beatname_uc} is the primary userspace daemon for -receiving audit events and managing the rules. Only a single process can receive -audit events through the "unicast" connection so any other daemons should be -stopped (e.g. stop `auditd`). -+ -`multicast` can be used in kernel versions 3.16 and newer. By using `multicast` -{beatname_uc} will receive an audit event broadcast that is not exclusive to a -a single process. This is ideal for situations where `auditd` is running and -managing the rules. -+ -By default {beatname_uc} will use `multicast` if the kernel version is 3.16 or -newer and no rules have been defined. Otherwise `unicast` will be used. - -*`immutable`*:: This boolean setting sets the audit config as immutable (`-e 2`). -This option can only be used with the `socket_type: unicast` since {beatname_uc} -needs to manage the rules to be able to set it. -+ -It is important to note that with this setting enabled, if {beatname_uc} is -stopped and resumed events will continue to be processed but the -configuration won't be updated until the system is restarted entirely. - -*`resolve_ids`*:: This boolean setting enables the resolution of UIDs and -GIDs to their associated names. The default value is true. - -*`failure_mode`*:: This determines the kernel's behavior on critical -failures such as errors sending events to {beatname_uc}, the backlog limit was -exceeded, the kernel ran out of memory, or the rate limit was exceeded. The -options are `silent`, `log`, or `panic`. `silent` basically makes the kernel -ignore the errors, `log` makes the kernel write the audit messages using -`printk` so they show up in system's syslog, and `panic` causes the kernel to -panic to prevent use of the machine. {beatname_uc}'s default is `silent`. - -*`backlog_limit`*:: This controls the maximum number of audit messages -that will be buffered by the kernel. - -*`rate_limit`*:: This sets a rate limit on the number of messages/sec -delivered by the kernel. The default is 0, which disables rate limiting. -Changing this value to anything other than zero can cause messages to be lost. -The preferred approach to reduce the messaging rate is be more selective in the -audit ruleset. - -*`include_raw_message`*:: This boolean setting causes {beatname_uc} to -include each of the raw messages that contributed to the event in the document -as a field called `event.original`. The default value is false. This setting is -primarily used for development and debugging purposes. - -*`include_warnings`*:: This boolean setting causes {beatname_uc} to -include as warnings any issues that were encountered while parsing the raw -messages. The messages are written to the `error.message` field. The default -value is false. When this setting is enabled the raw messages will be included -in the event regardless of the `include_raw_message` config setting. This -setting is primarily used for development and debugging purposes. - -*`audit_rules`*:: A string containing the audit rules that should be -installed to the kernel. There should be one rule per line. Comments can be -embedded in the string using `#` as a prefix. The format for rules is the same -used by the Linux `auditctl` utility. {beatname_uc} supports adding file watches -(`-w`) and syscall rules (`-a` or `-A`). For more information, see -<>. - -*`audit_rule_files`*:: A list of files to load audit rules from. This files are -loaded after the rules declared in `audit_rules` are loaded. Wildcards are -supported and will expand in lexicographical order. The format is the same as -that of the `audit_rules` field. - -*`ignore_errors`*:: This setting allows errors during rule loading and parsing -to be ignored, but logged as warnings. - -*`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to -prevent backpressure from propagating to the kernel and impacting audited -processes. -+ --- -The possible values are: - -- `auto` (default): {beatname_uc} uses the `kernel` strategy, if supported, or -falls back to the `userspace` strategy. -- `kernel`: {beatname_uc} sets the `backlog_wait_time` in the kernel's -audit framework to 0. This causes events to be discarded in the kernel if -the audit backlog queue fills to capacity. Requires a 3.14 kernel or -newer. -- `userspace`: {beatname_uc} drops events when there is backpressure -from the publishing pipeline. If no `rate_limit` is set, {beatname_uc} sets a rate -limit of 5000. Users should test their setup and adjust the `rate_limit` -option accordingly. -- `both`: {beatname_uc} uses the `kernel` and `userspace` strategies at the same -time. -- `none`: No backpressure mitigation measures are enabled. --- - -include::{docdir}/auditbeat-options.asciidoc[] - -[float] -[[audit-rules]] -=== Audit rules - -The audit rules are where you configure the activities that are audited. These -rules are configured as either syscalls or files that should be monitored. For -example you can track all `connect` syscalls or file system writes to -`/etc/passwd`. - -Auditing a large number of syscalls can place a heavy load on the system so -consider carefully the rules you define and try to apply filters in the rules -themselves to be as selective as possible. - -The kernel evaluates the rules in the order in which they were defined so place -the most active rules first in order to speed up evaluation. - -You can assign keys to each rule for better identification of the rule that -triggered an event and easier filtering later in Elasticsearch. - -Defining any audit rules in the config causes {beatname_uc} to purge all -existing audit rules prior to adding the rules specified in the config. -Therefore it is unnecessary and unsupported to include a `-D` (delete all) rule. - -["source","sh",subs="attributes"] ----- -{beatname_lc}.modules: -- module: auditd - audit_rules: | - # Things that affect identity. - -w /etc/group -p wa -k identity - -w /etc/passwd -p wa -k identity - -w /etc/gshadow -p wa -k identity - -w /etc/shadow -p wa -k identity - - # Unauthorized access attempts to files (unsuccessful). - -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access - -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access - -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access - -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access ----- - - -[float] -=== Example configuration - -The Auditd module supports the common configuration options that are -described under <>. Here -is an example configuration: - -[source,yaml] ----- -auditbeat.modules: -- module: auditd - # Load audit rules from separate files. Same format as audit.rules(7). - audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ] - audit_rules: | - ## Define audit rules here. - ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these - ## examples or add your own rules. - - ## If you are on a 64 bit platform, everything should be running - ## in 64 bit mode. This rule will detect any use of the 32 bit syscalls - ## because this might be a sign of someone exploiting a hole in the 32 - ## bit API. - #-a always,exit -F arch=b32 -S all -F key=32bit-abi - - ## Executions. - #-a always,exit -F arch=b64 -S execve,execveat -k exec - - ## External access (warning: these can be expensive to audit). - #-a always,exit -F arch=b64 -S accept,bind,connect -F key=external-access - - ## Identity changes. - #-w /etc/group -p wa -k identity - #-w /etc/passwd -p wa -k identity - #-w /etc/gshadow -p wa -k identity - - ## Unauthorized access attempts. - #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access - #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access - - ----- - - -:modulename!: - diff --git a/auditbeat/docs/modules/file_integrity.asciidoc b/auditbeat/docs/modules/file_integrity.asciidoc deleted file mode 100644 index 5257099270b..00000000000 --- a/auditbeat/docs/modules/file_integrity.asciidoc +++ /dev/null @@ -1,181 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -:modulename: file_integrity - -[id="{beatname_lc}-module-file_integrity"] -== File Integrity Module - -The `file_integrity` module sends events when a file is changed (created, -updated, or deleted) on disk. The events contain file metadata and hashes. - -The module is implemented for Linux, macOS (Darwin), and Windows. - -[float] -=== How it works - -This module uses features of the operating system to monitor file changes in -realtime. When the module starts it creates a subscription with the OS to -receive notifications of changes to the specified files or directories. Upon -receiving notification of a change the module will read the file's metadata -and the compute a hash of the file's contents. - -At startup this module will perform an initial scan of the configured files -and directories to generate baseline data for the monitored paths and detect -changes since the last time it was run. It uses locally persisted data in order -to only send events for new or modified files. - -The operating system features that power this feature are as follows. - -* Linux - Multiple backends are supported: `auto`, `fsnotify`, `kprobes`, `ebpf`. -By default, `fsnotify` is used, and therefore the kernel must have inotify support. -Inotify was initially merged into the 2.6.13 Linux kernel. -The eBPF backend uses modern eBPF features and supports 5.10.16+ kernels. -The `Kprobes` backend uses tracefs and supports 3.10+ kernels. -FSNotify doesn't have the ability to associate user data to file events. -The preferred backend can be selected by specifying the `backend` config option. -Since eBPF and Kprobes are in technical preview, `auto` will default to `fsnotify`. -* macOS (Darwin) - Uses the `FSEvents` API, present since macOS 10.5. This API -coalesces multiple changes to a file into a single event. {beatname_uc} translates -this coalesced changes into a meaningful sequence of actions. However, -in rare situations the reported events may have a different ordering than what -actually happened. -* Windows - `ReadDirectoryChangesW` is used. - -The file integrity module should not be used to monitor paths on network file -systems. - -[float] -=== Configuration options - -This module has some configuration options for tuning its behavior. The -following example shows all configuration options with their default values for -Linux. - -[source,yaml] ----- -- module: file_integrity - paths: - - /bin - - /usr/bin - - /sbin - - /usr/sbin - - /etc - exclude_files: - - '(?i)\.sw[nop]$' - - '~$' - - '/\.git($|/)' - include_files: [] - scan_at_start: true - scan_rate_per_sec: 50 MiB - max_file_size: 100 MiB - hash_types: [sha1] - recursive: false ----- - -This module also supports the -<> -described later. - -*`paths`*:: A list of paths (directories or files) to watch. Globs are -not supported. The specified paths should exist when the metricset is started. -Paths should be absolute, although the file integrity module will attempt to -resolve relative path events to their absolute file path. Symbolic links will -be resolved on module start and the link target will be watched if link resolution -is successful. Changes to the symbolic link after module start will not change -the watch target. If the link does not resolve to a valid target, the symbolic -link itself will be watched; if the symlink target becomes valid after module -start up this will not be picked up by the file system watches. - -*`exclude_files`*:: A list of regular expressions used to filter out events -for unwanted files. The expressions are matched against the full path of every -file and directory. When used in conjunction with `include_files`, file paths need -to match both `include_files` and not match `exclude_files` to be selected. -By default, no files are excluded. See <> -for a list of supported regexp patterns. It is recommended to wrap regular -expressions in single quotation marks to avoid issues with YAML escaping -rules. - -*`include_files`*:: A list of regular expressions used to specify which files to -select. When configured, only files matching the pattern will be monitored. -The expressions are matched against the full path of every file and directory. -When used in conjunction with `exclude_files`, file paths need -to match both `include_files` and not match `exclude_files` to be selected. -By default, all files are selected. See <> -for a list of supported regexp patterns. It is recommended to wrap regular -expressions in single quotation marks to avoid issues with YAML escaping -rules. - -*`scan_at_start`*:: A boolean value that controls if {beatname_uc} scans -over the configured file paths at startup and send events for the files -that have been modified since the last time {beatname_uc} was running. The -default value is true. -+ -This feature depends on data stored locally in `path.data` in order to determine -if a file has changed. The first time {beatname_uc} runs it will send an event -for each file it encounters. - -*`scan_rate_per_sec`*:: When `scan_at_start` is enabled this sets an -average read rate defined in bytes per second for the initial scan. This -throttles the amount of CPU and I/O that {beatname_uc} consumes at startup. -The default value is "50 MiB". Setting the value to "0" disables throttling. -For convenience units can be specified as a suffix to the value. The supported -units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, -`pib`, `pb`, `eib`, and `eb`. - -*`max_file_size`*:: The maximum size of a file in bytes for which -{beatname_uc} will compute hashes and run file parsers. Files larger than this -size will not be hashed or analysed by configured file parsers. The default -value is 100 MiB. For convenience, units can be specified as a suffix to the -value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, -`gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. - -*`hash_types`*:: A list of hash types to compute when the file changes. -The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`, -`sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha512_224`, `sha512_256`, -`sha3_224`, `sha3_256`, `sha3_384`, `sha3_512`, and `xxh64`. The default value is `sha1`. - -*`file_parsers`*:: A list of `file_integrity` fields under `file` that will be -populated by file format parsers. The available fields that can be analysed -are listed in the auditbeat.reference.yml file. File parsers are run on all -files within the `max_file_size` limit in the configured paths during a scan or -when a file event involves the file. Files that are not targets of the specific -file parser are only sniffed to examine whether analysis should proceed. This will -usually only involve reading a small number of bytes. - -*`recursive`*:: By default, the watches set to the paths specified in -`paths` are not recursive. This means that only changes to the contents -of this directories are watched. If `recursive` is set to `true`, the -`file_integrity` module will watch for changes on this directories and all -their subdirectories. - -*`backend`*:: (*Linux only*) Select the backend which will be used to -source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`. - -include::{docdir}/auditbeat-options.asciidoc[] - - -[float] -=== Example configuration - -The File Integrity module supports the common configuration options that are -described under <>. Here -is an example configuration: - -[source,yaml] ----- -auditbeat.modules: -- module: file_integrity - paths: - - /bin - - /usr/bin - - /sbin - - /usr/sbin - - /etc - ----- - - -:modulename!: - diff --git a/x-pack/auditbeat/docs/modules/system.asciidoc b/x-pack/auditbeat/docs/modules/system.asciidoc deleted file mode 100644 index c17f90119dc..00000000000 --- a/x-pack/auditbeat/docs/modules/system.asciidoc +++ /dev/null @@ -1,211 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -:modulename: system - -[id="{beatname_lc}-module-system"] -[role="xpack"] -== System Module - -beta[] - -The `system` module collects various security related information about -a system. All datasets send both periodic state information (e.g. all currently -running processes) and real-time changes (e.g. when a new process starts -or stops). - -The module is fully implemented for Linux on x86. Currently, the `socket` module is not available on ARM. Some datasets are also available -for macOS (Darwin) and Windows. - -[float] -=== How it works - -Each dataset sends two kinds of information: state and events. - -State information is sent periodically and (for some datasets) on startup. -A state update will consist of one event per object that is currently -active on the system (e.g. a process). All events belonging to the same state -update will share the same UUID in `event.id`. - -The frequency of state updates can be controlled for all datasets using the -`state.period` configuration option. Overrides are available per dataset. -The default is `12h`. - -Event information is sent as the events occur (e.g. a process starts or stops). -All datasets are currently using a poll model to retrieve their data. -The frequency of these polls is controlled by the `period` configuration -parameter. - -[float] -==== Entity IDs - -This module populates `entity_id` fields to uniquely identify entities (users, -packages, processes...) within a host. This requires {beatname_uc} -to obtain a unique identifier for the host: - -- Windows: Uses the `HKLM\Software\Microsoft\Cryptography\MachineGuid` registry -key. -- macOS: Uses the value returned by `gethostuuid(2)` system call. -- Linux: Uses the content of one of the following files, created by either -`systemd` or `dbus`: - * /etc/machine-id - * /var/lib/dbus/machine-id - * /var/db/dbus/machine-id - -NOTE: Under CentOS 6.x, it's possible that none of the files above exist. -In that case, running `dbus-uuidgen --ensure` (provided by the `dbus` package) -will generate one for you. - -[float] -==== Example dashboard - -The module comes with a sample dashboard: - -[role="screenshot"] -image::./images/auditbeat-system-overview-dashboard.png[Auditbeat System Overview Dashboard] - -[float] -=== Configuration options - -This module has some configuration options for controlling its behavior. The -following example shows all configuration options with their default values for -Linux. - -NOTE: It is recommended to configure some datasets separately. See below for a -sample suggested configuration. - -[source,yaml] ----- -- module: system - datasets: - - host - - login - - package - - process - - socket - - user - period: 10s - state.period: 12h - - socket.include_localhost: false - - user.detect_password_changes: true ----- - -This module also supports the -<> -described later. - -*`state.period`*:: The interval at which the datasets send full state information. -This option can be overridden per dataset using `{dataset}.state.period`. - -*`user.detect_password_changes`*:: If the `user` dataset is configured and -this option is set to `true`, Auditbeat will read password information in `/etc/passwd` -and `/etc/shadow` to detect password changes. A hash will be kept locally in -the `beat.db` file to detect changes between Auditbeat restarts. The `beat.db` file -should be readable only by the root user and be treated similar to the shadow file -itself. - -include::{docdir}/auditbeat-options.asciidoc[] - -[float] -=== Suggested configuration - -Processes and sockets can be short-lived, so the chance of missing an update -increases if the polling interval is too large. - -On the other hand, host and user information is unlikely to change frequently, -so a longer polling interval can be used. - -[source,yaml] ----- -- module: system - datasets: - - host - - login - - package - - user - period: 1m - - user.detect_password_changes: true - -- module: system - datasets: - - process - - socket - period: 1s ----- - - -[float] -=== Example configuration - -The System module supports the common configuration options that are -described under <>. Here -is an example configuration: - -[source,yaml] ----- -auditbeat.modules: -- module: system - datasets: - - package # Installed, updated, and removed packages - - period: 2m # The frequency at which the datasets check for changes - -- module: system - datasets: - - host # General host information, e.g. uptime, IPs - - login # User logins, logouts, and system boots. - - process # Started and stopped processes - - socket # Opened and closed sockets - - user # User information - - # How often datasets send state updates with the - # current state of the system (e.g. all currently - # running processes, all open sockets). - state.period: 12h - - # Enabled by default. Auditbeat will read password fields in - # /etc/passwd and /etc/shadow and store a hash locally to - # detect any changes. - user.detect_password_changes: true - - # File patterns of the login record files. - login.wtmp_file_pattern: /var/log/wtmp* - login.btmp_file_pattern: /var/log/btmp* ----- - - -:modulename!: - -[float] -=== Datasets - -The following datasets are available: - -* <<{beatname_lc}-dataset-system-host,host>> - -* <<{beatname_lc}-dataset-system-login,login>> - -* <<{beatname_lc}-dataset-system-package,package>> - -* <<{beatname_lc}-dataset-system-process,process>> - -* <<{beatname_lc}-dataset-system-socket,socket>> - -* <<{beatname_lc}-dataset-system-user,user>> - -include::system/host.asciidoc[] - -include::system/login.asciidoc[] - -include::system/package.asciidoc[] - -include::system/process.asciidoc[] - -include::system/socket.asciidoc[] - -include::system/user.asciidoc[] - diff --git a/x-pack/auditbeat/docs/modules/system/host.asciidoc b/x-pack/auditbeat/docs/modules/system/host.asciidoc deleted file mode 100644 index 9c36068927f..00000000000 --- a/x-pack/auditbeat/docs/modules/system/host.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[id="{beatname_lc}-dataset-system-host"] -=== System host dataset - -include::../../../module/system/host/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the dataset, see the -<> section. - -Here is an example document generated by this dataset: - -[source,json] ----- -include::../../../module/system/host/_meta/data.json[] ----- diff --git a/x-pack/auditbeat/docs/modules/system/login.asciidoc b/x-pack/auditbeat/docs/modules/system/login.asciidoc deleted file mode 100644 index d042abb7add..00000000000 --- a/x-pack/auditbeat/docs/modules/system/login.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[id="{beatname_lc}-dataset-system-login"] -=== System login dataset - -include::../../../module/system/login/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the dataset, see the -<> section. - -Here is an example document generated by this dataset: - -[source,json] ----- -include::../../../module/system/login/_meta/data.json[] ----- diff --git a/x-pack/auditbeat/docs/modules/system/package.asciidoc b/x-pack/auditbeat/docs/modules/system/package.asciidoc deleted file mode 100644 index ec87bd976da..00000000000 --- a/x-pack/auditbeat/docs/modules/system/package.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[id="{beatname_lc}-dataset-system-package"] -=== System package dataset - -include::../../../module/system/package/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the dataset, see the -<> section. - -Here is an example document generated by this dataset: - -[source,json] ----- -include::../../../module/system/package/_meta/data.json[] ----- diff --git a/x-pack/auditbeat/docs/modules/system/process.asciidoc b/x-pack/auditbeat/docs/modules/system/process.asciidoc deleted file mode 100644 index a015dfc29af..00000000000 --- a/x-pack/auditbeat/docs/modules/system/process.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[id="{beatname_lc}-dataset-system-process"] -=== System process dataset - -include::../../../module/system/process/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the dataset, see the -<> section. - -Here is an example document generated by this dataset: - -[source,json] ----- -include::../../../module/system/process/_meta/data.json[] ----- diff --git a/x-pack/auditbeat/docs/modules/system/socket.asciidoc b/x-pack/auditbeat/docs/modules/system/socket.asciidoc deleted file mode 100644 index 387756c1ca9..00000000000 --- a/x-pack/auditbeat/docs/modules/system/socket.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[id="{beatname_lc}-dataset-system-socket"] -=== System socket dataset - -include::../../../module/system/socket/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the dataset, see the -<> section. - -Here is an example document generated by this dataset: - -[source,json] ----- -include::../../../module/system/socket/_meta/data.json[] ----- diff --git a/x-pack/auditbeat/docs/modules/system/user.asciidoc b/x-pack/auditbeat/docs/modules/system/user.asciidoc deleted file mode 100644 index 58b3a07e3af..00000000000 --- a/x-pack/auditbeat/docs/modules/system/user.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[id="{beatname_lc}-dataset-system-user"] -=== System user dataset - -include::../../../module/system/user/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the dataset, see the -<> section. - -Here is an example document generated by this dataset: - -[source,json] ----- -include::../../../module/system/user/_meta/data.json[] ----- From 7aa47018abcef456facf6d6202e8252a16d099d6 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 17:15:56 +0300 Subject: [PATCH 09/34] fixing also request for http gw --- auditbeat/docs/modules/auditd.asciidoc | 327 ++++++++++++++++++ .../docs/modules/file_integrity.asciidoc | 181 ++++++++++ go.mod | 1 + go.sum | 2 + x-pack/auditbeat/docs/modules/system.asciidoc | 211 +++++++++++ .../docs/modules/system/host.asciidoc | 21 ++ .../docs/modules/system/login.asciidoc | 21 ++ .../docs/modules/system/package.asciidoc | 21 ++ .../docs/modules/system/process.asciidoc | 21 ++ .../docs/modules/system/socket.asciidoc | 21 ++ .../docs/modules/system/user.asciidoc | 21 ++ .../module/aws/cloudwatch/cloudwatch.go | 32 +- x-pack/metricbeat/module/aws/utils.go | 27 +- 13 files changed, 894 insertions(+), 13 deletions(-) create mode 100644 auditbeat/docs/modules/auditd.asciidoc create mode 100644 auditbeat/docs/modules/file_integrity.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system/host.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system/login.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system/package.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system/process.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system/socket.asciidoc create mode 100644 x-pack/auditbeat/docs/modules/system/user.asciidoc diff --git a/auditbeat/docs/modules/auditd.asciidoc b/auditbeat/docs/modules/auditd.asciidoc new file mode 100644 index 00000000000..0361dc56097 --- /dev/null +++ b/auditbeat/docs/modules/auditd.asciidoc @@ -0,0 +1,327 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +:modulename: auditd + +[id="{beatname_lc}-module-auditd"] +== Auditd Module + +The `auditd` module receives audit events from the Linux Audit Framework that +is a part of the Linux kernel. + +This module is available only for Linux. + +[float] +=== How it works + +This module establishes a subscription to the kernel to receive the events +as they occur. So unlike most other modules, the `period` configuration +option is unused because it is not implemented using polling. + +The Linux Audit Framework can send multiple messages for a single auditable +event. For example, a `rename` syscall causes the kernel to send eight separate +messages. Each message describes a different aspect of the activity that is +occurring (the syscall itself, file paths, current working directory, process +title). This module will combine all of the data from each of the messages +into a single event. + +Messages for one event can be interleaved with messages from another event. This +module will buffer the messages in order to combine related messages into a +single event even if they arrive interleaved or out of order. + +[float] +=== Useful commands + +When running {beatname_uc} with the `auditd` module enabled, you might find +that other monitoring tools interfere with {beatname_uc}. + +For example, you might encounter errors if another process, such as `auditd`, is +registered to receive data from the Linux Audit Framework. You can use these +commands to see if the `auditd` service is running and stop it: + +* See if `auditd` is running: ++ +[source,shell] +----- +service auditd status +----- + +* Stop the `auditd` service: ++ +[source,shell] +----- +service auditd stop +----- + +* Disable `auditd` from starting on boot: ++ +[source,shell] +----- +chkconfig auditd off +----- + +To save CPU usage and disk space, you can use this command to stop `journald` +from listening to audit messages: + +[source,shell] +----- +systemctl mask systemd-journald-audit.socket +----- + +[float] +=== Inspect the kernel audit system status + +{beatname_uc} provides useful commands to query the state of the audit system +in the Linux kernel. + +* See the list of installed audit rules: ++ +[source,shell] +----- +auditbeat show auditd-rules +----- ++ +Prints the list of loaded rules, similar to `auditctl -l`: ++ +[source,shell] +----- +-a never,exit -S all -F pid=26253 +-a always,exit -F arch=b32 -S all -F key=32bit-abi +-a always,exit -F arch=b64 -S execve,execveat -F key=exec +-a always,exit -F arch=b64 -S connect,accept,bind -F key=external-access +-w /etc/group -p wa -k identity +-w /etc/passwd -p wa -k identity +-w /etc/gshadow -p wa -k identity +-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F key=access +-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F key=access +----- + +* See the status of the audit system: ++ +[source,shell] +----- +auditbeat show auditd-status +----- ++ +Prints the status of the kernel audit system, similar to `auditctl -s`: ++ +[source,shell] +----- +enabled 1 +failure 0 +pid 0 +rate_limit 0 +backlog_limit 8192 +lost 14407 +backlog 0 +backlog_wait_time 0 +features 0xf +----- + +[float] +=== Configuration options + +This module has some configuration options for tuning its behavior. The +following example shows all configuration options with their default values. + +[source,yaml] +---- +- module: auditd + resolve_ids: true + failure_mode: silent + backlog_limit: 8192 + rate_limit: 0 + include_raw_message: false + include_warnings: false + backpressure_strategy: auto + immutable: false +---- + +This module also supports the +<> +described later. + +*`socket_type`*:: This optional setting controls the type of +socket that {beatname_uc} uses to receive events from the kernel. The two +options are `unicast` and `multicast`. ++ +`unicast` should be used when {beatname_uc} is the primary userspace daemon for +receiving audit events and managing the rules. Only a single process can receive +audit events through the "unicast" connection so any other daemons should be +stopped (e.g. stop `auditd`). ++ +`multicast` can be used in kernel versions 3.16 and newer. By using `multicast` +{beatname_uc} will receive an audit event broadcast that is not exclusive to a +a single process. This is ideal for situations where `auditd` is running and +managing the rules. ++ +By default {beatname_uc} will use `multicast` if the kernel version is 3.16 or +newer and no rules have been defined. Otherwise `unicast` will be used. + +*`immutable`*:: This boolean setting sets the audit config as immutable (`-e 2`). +This option can only be used with the `socket_type: unicast` since {beatname_uc} +needs to manage the rules to be able to set it. ++ +It is important to note that with this setting enabled, if {beatname_uc} is +stopped and resumed events will continue to be processed but the +configuration won't be updated until the system is restarted entirely. + +*`resolve_ids`*:: This boolean setting enables the resolution of UIDs and +GIDs to their associated names. The default value is true. + +*`failure_mode`*:: This determines the kernel's behavior on critical +failures such as errors sending events to {beatname_uc}, the backlog limit was +exceeded, the kernel ran out of memory, or the rate limit was exceeded. The +options are `silent`, `log`, or `panic`. `silent` basically makes the kernel +ignore the errors, `log` makes the kernel write the audit messages using +`printk` so they show up in system's syslog, and `panic` causes the kernel to +panic to prevent use of the machine. {beatname_uc}'s default is `silent`. + +*`backlog_limit`*:: This controls the maximum number of audit messages +that will be buffered by the kernel. + +*`rate_limit`*:: This sets a rate limit on the number of messages/sec +delivered by the kernel. The default is 0, which disables rate limiting. +Changing this value to anything other than zero can cause messages to be lost. +The preferred approach to reduce the messaging rate is be more selective in the +audit ruleset. + +*`include_raw_message`*:: This boolean setting causes {beatname_uc} to +include each of the raw messages that contributed to the event in the document +as a field called `event.original`. The default value is false. This setting is +primarily used for development and debugging purposes. + +*`include_warnings`*:: This boolean setting causes {beatname_uc} to +include as warnings any issues that were encountered while parsing the raw +messages. The messages are written to the `error.message` field. The default +value is false. When this setting is enabled the raw messages will be included +in the event regardless of the `include_raw_message` config setting. This +setting is primarily used for development and debugging purposes. + +*`audit_rules`*:: A string containing the audit rules that should be +installed to the kernel. There should be one rule per line. Comments can be +embedded in the string using `#` as a prefix. The format for rules is the same +used by the Linux `auditctl` utility. {beatname_uc} supports adding file watches +(`-w`) and syscall rules (`-a` or `-A`). For more information, see +<>. + +*`audit_rule_files`*:: A list of files to load audit rules from. This files are +loaded after the rules declared in `audit_rules` are loaded. Wildcards are +supported and will expand in lexicographical order. The format is the same as +that of the `audit_rules` field. + +*`ignore_errors`*:: This setting allows errors during rule loading and parsing +to be ignored, but logged as warnings. + +*`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to +prevent backpressure from propagating to the kernel and impacting audited +processes. ++ +-- +The possible values are: + +- `auto` (default): {beatname_uc} uses the `kernel` strategy, if supported, or +falls back to the `userspace` strategy. +- `kernel`: {beatname_uc} sets the `backlog_wait_time` in the kernel's +audit framework to 0. This causes events to be discarded in the kernel if +the audit backlog queue fills to capacity. Requires a 3.14 kernel or +newer. +- `userspace`: {beatname_uc} drops events when there is backpressure +from the publishing pipeline. If no `rate_limit` is set, {beatname_uc} sets a rate +limit of 5000. Users should test their setup and adjust the `rate_limit` +option accordingly. +- `both`: {beatname_uc} uses the `kernel` and `userspace` strategies at the same +time. +- `none`: No backpressure mitigation measures are enabled. +-- + +include::{docdir}/auditbeat-options.asciidoc[] + +[float] +[[audit-rules]] +=== Audit rules + +The audit rules are where you configure the activities that are audited. These +rules are configured as either syscalls or files that should be monitored. For +example you can track all `connect` syscalls or file system writes to +`/etc/passwd`. + +Auditing a large number of syscalls can place a heavy load on the system so +consider carefully the rules you define and try to apply filters in the rules +themselves to be as selective as possible. + +The kernel evaluates the rules in the order in which they were defined so place +the most active rules first in order to speed up evaluation. + +You can assign keys to each rule for better identification of the rule that +triggered an event and easier filtering later in Elasticsearch. + +Defining any audit rules in the config causes {beatname_uc} to purge all +existing audit rules prior to adding the rules specified in the config. +Therefore it is unnecessary and unsupported to include a `-D` (delete all) rule. + +["source","sh",subs="attributes"] +---- +{beatname_lc}.modules: +- module: auditd + audit_rules: | + # Things that affect identity. + -w /etc/group -p wa -k identity + -w /etc/passwd -p wa -k identity + -w /etc/gshadow -p wa -k identity + -w /etc/shadow -p wa -k identity + + # Unauthorized access attempts to files (unsuccessful). + -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access + -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access + -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access + -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access +---- + + +[float] +=== Example configuration + +The Auditd module supports the common configuration options that are +described under <>. Here +is an example configuration: + +[source,yaml] +---- +auditbeat.modules: +- module: auditd + # Load audit rules from separate files. Same format as audit.rules(7). + audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ] + audit_rules: | + ## Define audit rules here. + ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these + ## examples or add your own rules. + + ## If you are on a 64 bit platform, everything should be running + ## in 64 bit mode. This rule will detect any use of the 32 bit syscalls + ## because this might be a sign of someone exploiting a hole in the 32 + ## bit API. + #-a always,exit -F arch=b32 -S all -F key=32bit-abi + + ## Executions. + #-a always,exit -F arch=b64 -S execve,execveat -k exec + + ## External access (warning: these can be expensive to audit). + #-a always,exit -F arch=b64 -S accept,bind,connect -F key=external-access + + ## Identity changes. + #-w /etc/group -p wa -k identity + #-w /etc/passwd -p wa -k identity + #-w /etc/gshadow -p wa -k identity + + ## Unauthorized access attempts. + #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access + #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access + + +---- + + +:modulename!: + diff --git a/auditbeat/docs/modules/file_integrity.asciidoc b/auditbeat/docs/modules/file_integrity.asciidoc new file mode 100644 index 00000000000..5257099270b --- /dev/null +++ b/auditbeat/docs/modules/file_integrity.asciidoc @@ -0,0 +1,181 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +:modulename: file_integrity + +[id="{beatname_lc}-module-file_integrity"] +== File Integrity Module + +The `file_integrity` module sends events when a file is changed (created, +updated, or deleted) on disk. The events contain file metadata and hashes. + +The module is implemented for Linux, macOS (Darwin), and Windows. + +[float] +=== How it works + +This module uses features of the operating system to monitor file changes in +realtime. When the module starts it creates a subscription with the OS to +receive notifications of changes to the specified files or directories. Upon +receiving notification of a change the module will read the file's metadata +and the compute a hash of the file's contents. + +At startup this module will perform an initial scan of the configured files +and directories to generate baseline data for the monitored paths and detect +changes since the last time it was run. It uses locally persisted data in order +to only send events for new or modified files. + +The operating system features that power this feature are as follows. + +* Linux - Multiple backends are supported: `auto`, `fsnotify`, `kprobes`, `ebpf`. +By default, `fsnotify` is used, and therefore the kernel must have inotify support. +Inotify was initially merged into the 2.6.13 Linux kernel. +The eBPF backend uses modern eBPF features and supports 5.10.16+ kernels. +The `Kprobes` backend uses tracefs and supports 3.10+ kernels. +FSNotify doesn't have the ability to associate user data to file events. +The preferred backend can be selected by specifying the `backend` config option. +Since eBPF and Kprobes are in technical preview, `auto` will default to `fsnotify`. +* macOS (Darwin) - Uses the `FSEvents` API, present since macOS 10.5. This API +coalesces multiple changes to a file into a single event. {beatname_uc} translates +this coalesced changes into a meaningful sequence of actions. However, +in rare situations the reported events may have a different ordering than what +actually happened. +* Windows - `ReadDirectoryChangesW` is used. + +The file integrity module should not be used to monitor paths on network file +systems. + +[float] +=== Configuration options + +This module has some configuration options for tuning its behavior. The +following example shows all configuration options with their default values for +Linux. + +[source,yaml] +---- +- module: file_integrity + paths: + - /bin + - /usr/bin + - /sbin + - /usr/sbin + - /etc + exclude_files: + - '(?i)\.sw[nop]$' + - '~$' + - '/\.git($|/)' + include_files: [] + scan_at_start: true + scan_rate_per_sec: 50 MiB + max_file_size: 100 MiB + hash_types: [sha1] + recursive: false +---- + +This module also supports the +<> +described later. + +*`paths`*:: A list of paths (directories or files) to watch. Globs are +not supported. The specified paths should exist when the metricset is started. +Paths should be absolute, although the file integrity module will attempt to +resolve relative path events to their absolute file path. Symbolic links will +be resolved on module start and the link target will be watched if link resolution +is successful. Changes to the symbolic link after module start will not change +the watch target. If the link does not resolve to a valid target, the symbolic +link itself will be watched; if the symlink target becomes valid after module +start up this will not be picked up by the file system watches. + +*`exclude_files`*:: A list of regular expressions used to filter out events +for unwanted files. The expressions are matched against the full path of every +file and directory. When used in conjunction with `include_files`, file paths need +to match both `include_files` and not match `exclude_files` to be selected. +By default, no files are excluded. See <> +for a list of supported regexp patterns. It is recommended to wrap regular +expressions in single quotation marks to avoid issues with YAML escaping +rules. + +*`include_files`*:: A list of regular expressions used to specify which files to +select. When configured, only files matching the pattern will be monitored. +The expressions are matched against the full path of every file and directory. +When used in conjunction with `exclude_files`, file paths need +to match both `include_files` and not match `exclude_files` to be selected. +By default, all files are selected. See <> +for a list of supported regexp patterns. It is recommended to wrap regular +expressions in single quotation marks to avoid issues with YAML escaping +rules. + +*`scan_at_start`*:: A boolean value that controls if {beatname_uc} scans +over the configured file paths at startup and send events for the files +that have been modified since the last time {beatname_uc} was running. The +default value is true. ++ +This feature depends on data stored locally in `path.data` in order to determine +if a file has changed. The first time {beatname_uc} runs it will send an event +for each file it encounters. + +*`scan_rate_per_sec`*:: When `scan_at_start` is enabled this sets an +average read rate defined in bytes per second for the initial scan. This +throttles the amount of CPU and I/O that {beatname_uc} consumes at startup. +The default value is "50 MiB". Setting the value to "0" disables throttling. +For convenience units can be specified as a suffix to the value. The supported +units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, +`pib`, `pb`, `eib`, and `eb`. + +*`max_file_size`*:: The maximum size of a file in bytes for which +{beatname_uc} will compute hashes and run file parsers. Files larger than this +size will not be hashed or analysed by configured file parsers. The default +value is 100 MiB. For convenience, units can be specified as a suffix to the +value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, +`gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. + +*`hash_types`*:: A list of hash types to compute when the file changes. +The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`, +`sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha512_224`, `sha512_256`, +`sha3_224`, `sha3_256`, `sha3_384`, `sha3_512`, and `xxh64`. The default value is `sha1`. + +*`file_parsers`*:: A list of `file_integrity` fields under `file` that will be +populated by file format parsers. The available fields that can be analysed +are listed in the auditbeat.reference.yml file. File parsers are run on all +files within the `max_file_size` limit in the configured paths during a scan or +when a file event involves the file. Files that are not targets of the specific +file parser are only sniffed to examine whether analysis should proceed. This will +usually only involve reading a small number of bytes. + +*`recursive`*:: By default, the watches set to the paths specified in +`paths` are not recursive. This means that only changes to the contents +of this directories are watched. If `recursive` is set to `true`, the +`file_integrity` module will watch for changes on this directories and all +their subdirectories. + +*`backend`*:: (*Linux only*) Select the backend which will be used to +source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`. + +include::{docdir}/auditbeat-options.asciidoc[] + + +[float] +=== Example configuration + +The File Integrity module supports the common configuration options that are +described under <>. Here +is an example configuration: + +[source,yaml] +---- +auditbeat.modules: +- module: file_integrity + paths: + - /bin + - /usr/bin + - /sbin + - /usr/sbin + - /etc + +---- + + +:modulename!: + diff --git a/go.mod b/go.mod index 9fdd2a93d18..77d16fcc910 100644 --- a/go.mod +++ b/go.mod @@ -185,6 +185,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13 github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8 + github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5 github.com/aws/aws-sdk-go-v2/service/health v1.26.4 github.com/aws/aws-sdk-go-v2/service/kinesis v1.29.5 diff --git a/go.sum b/go.sum index cb6c65ebe77..fb42b9e5205 100644 --- a/go.sum +++ b/go.sum @@ -329,6 +329,8 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGx github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8 h1:CgEyY7gfTf7lHYcCi7+w6jJ1XQBugjpadtsuN3TGxdQ= github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8/go.mod h1:z99ur4Ha5540t8hb5XtqV/UMOnEoEZK22lhr5ZBS0zw= +github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8 h1:SWBNBbVbThg5Hdi3hWbVaDFjV/OyPbuqZLu4N+mj/Es= +github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8/go.mod h1:lz2IT8gzzSwao0Pa6uMSdCIPsprmgCkW83q6sHGZFDw= github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5 h1:YeTVIy7cJLeahs7K0jQGDGAd1YYND/to/z8N3kqZBhY= github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5/go.mod h1:y45SdA9v+dLlweaqwAQMoFeXqdRvgwevafa2X8iTqZQ= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5 h1:/YvqO1j75i4leoV+Z3a5s/dAlEszf2wTKBW8jc3Gd4s= diff --git a/x-pack/auditbeat/docs/modules/system.asciidoc b/x-pack/auditbeat/docs/modules/system.asciidoc new file mode 100644 index 00000000000..c17f90119dc --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system.asciidoc @@ -0,0 +1,211 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +:modulename: system + +[id="{beatname_lc}-module-system"] +[role="xpack"] +== System Module + +beta[] + +The `system` module collects various security related information about +a system. All datasets send both periodic state information (e.g. all currently +running processes) and real-time changes (e.g. when a new process starts +or stops). + +The module is fully implemented for Linux on x86. Currently, the `socket` module is not available on ARM. Some datasets are also available +for macOS (Darwin) and Windows. + +[float] +=== How it works + +Each dataset sends two kinds of information: state and events. + +State information is sent periodically and (for some datasets) on startup. +A state update will consist of one event per object that is currently +active on the system (e.g. a process). All events belonging to the same state +update will share the same UUID in `event.id`. + +The frequency of state updates can be controlled for all datasets using the +`state.period` configuration option. Overrides are available per dataset. +The default is `12h`. + +Event information is sent as the events occur (e.g. a process starts or stops). +All datasets are currently using a poll model to retrieve their data. +The frequency of these polls is controlled by the `period` configuration +parameter. + +[float] +==== Entity IDs + +This module populates `entity_id` fields to uniquely identify entities (users, +packages, processes...) within a host. This requires {beatname_uc} +to obtain a unique identifier for the host: + +- Windows: Uses the `HKLM\Software\Microsoft\Cryptography\MachineGuid` registry +key. +- macOS: Uses the value returned by `gethostuuid(2)` system call. +- Linux: Uses the content of one of the following files, created by either +`systemd` or `dbus`: + * /etc/machine-id + * /var/lib/dbus/machine-id + * /var/db/dbus/machine-id + +NOTE: Under CentOS 6.x, it's possible that none of the files above exist. +In that case, running `dbus-uuidgen --ensure` (provided by the `dbus` package) +will generate one for you. + +[float] +==== Example dashboard + +The module comes with a sample dashboard: + +[role="screenshot"] +image::./images/auditbeat-system-overview-dashboard.png[Auditbeat System Overview Dashboard] + +[float] +=== Configuration options + +This module has some configuration options for controlling its behavior. The +following example shows all configuration options with their default values for +Linux. + +NOTE: It is recommended to configure some datasets separately. See below for a +sample suggested configuration. + +[source,yaml] +---- +- module: system + datasets: + - host + - login + - package + - process + - socket + - user + period: 10s + state.period: 12h + + socket.include_localhost: false + + user.detect_password_changes: true +---- + +This module also supports the +<> +described later. + +*`state.period`*:: The interval at which the datasets send full state information. +This option can be overridden per dataset using `{dataset}.state.period`. + +*`user.detect_password_changes`*:: If the `user` dataset is configured and +this option is set to `true`, Auditbeat will read password information in `/etc/passwd` +and `/etc/shadow` to detect password changes. A hash will be kept locally in +the `beat.db` file to detect changes between Auditbeat restarts. The `beat.db` file +should be readable only by the root user and be treated similar to the shadow file +itself. + +include::{docdir}/auditbeat-options.asciidoc[] + +[float] +=== Suggested configuration + +Processes and sockets can be short-lived, so the chance of missing an update +increases if the polling interval is too large. + +On the other hand, host and user information is unlikely to change frequently, +so a longer polling interval can be used. + +[source,yaml] +---- +- module: system + datasets: + - host + - login + - package + - user + period: 1m + + user.detect_password_changes: true + +- module: system + datasets: + - process + - socket + period: 1s +---- + + +[float] +=== Example configuration + +The System module supports the common configuration options that are +described under <>. Here +is an example configuration: + +[source,yaml] +---- +auditbeat.modules: +- module: system + datasets: + - package # Installed, updated, and removed packages + + period: 2m # The frequency at which the datasets check for changes + +- module: system + datasets: + - host # General host information, e.g. uptime, IPs + - login # User logins, logouts, and system boots. + - process # Started and stopped processes + - socket # Opened and closed sockets + - user # User information + + # How often datasets send state updates with the + # current state of the system (e.g. all currently + # running processes, all open sockets). + state.period: 12h + + # Enabled by default. Auditbeat will read password fields in + # /etc/passwd and /etc/shadow and store a hash locally to + # detect any changes. + user.detect_password_changes: true + + # File patterns of the login record files. + login.wtmp_file_pattern: /var/log/wtmp* + login.btmp_file_pattern: /var/log/btmp* +---- + + +:modulename!: + +[float] +=== Datasets + +The following datasets are available: + +* <<{beatname_lc}-dataset-system-host,host>> + +* <<{beatname_lc}-dataset-system-login,login>> + +* <<{beatname_lc}-dataset-system-package,package>> + +* <<{beatname_lc}-dataset-system-process,process>> + +* <<{beatname_lc}-dataset-system-socket,socket>> + +* <<{beatname_lc}-dataset-system-user,user>> + +include::system/host.asciidoc[] + +include::system/login.asciidoc[] + +include::system/package.asciidoc[] + +include::system/process.asciidoc[] + +include::system/socket.asciidoc[] + +include::system/user.asciidoc[] + diff --git a/x-pack/auditbeat/docs/modules/system/host.asciidoc b/x-pack/auditbeat/docs/modules/system/host.asciidoc new file mode 100644 index 00000000000..9c36068927f --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/host.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-host"] +=== System host dataset + +include::../../../module/system/host/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/host/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/docs/modules/system/login.asciidoc b/x-pack/auditbeat/docs/modules/system/login.asciidoc new file mode 100644 index 00000000000..d042abb7add --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/login.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-login"] +=== System login dataset + +include::../../../module/system/login/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/login/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/docs/modules/system/package.asciidoc b/x-pack/auditbeat/docs/modules/system/package.asciidoc new file mode 100644 index 00000000000..ec87bd976da --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/package.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-package"] +=== System package dataset + +include::../../../module/system/package/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/package/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/docs/modules/system/process.asciidoc b/x-pack/auditbeat/docs/modules/system/process.asciidoc new file mode 100644 index 00000000000..a015dfc29af --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/process.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-process"] +=== System process dataset + +include::../../../module/system/process/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/process/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/docs/modules/system/socket.asciidoc b/x-pack/auditbeat/docs/modules/system/socket.asciidoc new file mode 100644 index 00000000000..387756c1ca9 --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/socket.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-socket"] +=== System socket dataset + +include::../../../module/system/socket/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/socket/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/docs/modules/system/user.asciidoc b/x-pack/auditbeat/docs/modules/system/user.asciidoc new file mode 100644 index 00000000000..58b3a07e3af --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/user.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-user"] +=== System user dataset + +include::../../../module/system/user/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/user/_meta/data.json[] +---- diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 1024ccf9d31..3236d93c72b 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -13,11 +13,11 @@ import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/apigateway" + "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" resourcegroupstaggingapitypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" @@ -123,8 +123,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency) m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // Initialise the map that will be used in case APiGW api is configured + inforestapi := make(map[string]string) infoapi := make(map[string]string) - // Check statistic method in config err := m.checkStatistics() if err != nil { @@ -148,8 +148,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.logger.Debugf("Collecting metrics from AWS region %s", regionName) beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - - svcCloudwatch, svcResourceAPI, _, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + svcCloudwatch, svcResourceAPI, _, _, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName) } @@ -174,7 +173,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - svcCloudwatch, svcResourceAPI, svcRestAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + svcCloudwatch, svcResourceAPI, svcRestAPI, svcHttpAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName, err) continue @@ -192,7 +191,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for namespace, namespaceDetails := range namespaceDetailTotal { m.logger.Debugf("Collected metrics from namespace %s", namespace) - // filter listMetricsOutput by detailed configuration per each namespace filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespace, namespaceDetails) @@ -202,12 +200,20 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { //Check whether namespace is APIGW checkns := "AWS/ApiGateway" if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { - infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + // inforestapi inlcudes only Rest APIs + inforestapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } + // infoapi inlcudes only WebSocket and HTTP APIs + infoapi, err = aws.GetAPIsOutput(svcHttpAPI) + if err != nil { + m.Logger().Errorf("could not get http and websocket apis output: %v", err) + } + m.Logger().Debugf("inforestapi response: %v", inforestapi) + m.Logger().Debugf("infoapi response: %v", infoapi) } - m.Logger().Infof("PASSS InfoAPI response: %v", infoapi) + eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) @@ -231,7 +237,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // createAwsRequiredClients will return the two necessary client instances to do Metric requests to the AWS API -func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, *apigateway.Client, error) { +func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, *apigateway.Client, *apigatewayv2.Client, error) { m.logger.Debugf("Collecting metrics from AWS region %s", regionName) svcCloudwatchClient := cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) { @@ -251,7 +257,11 @@ func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionNa }) - return svcCloudwatchClient, svcResourceAPIClient, svcGetRestAPIClient, nil + svcGetHttpAPIClient := apigatewayv2.NewFromConfig(beatsConfig, func(o *apigatewayv2.Options) { + + }) + + return svcCloudwatchClient, svcResourceAPIClient, svcGetRestAPIClient, svcGetHttpAPIClient, nil } // filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match @@ -543,7 +553,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient delete(resourceTagMap, identifier) continue } - m.logger.Infof("In region %s, service %s tags match tags_filter", regionName, identifier) + m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifier) } for _, output := range metricDataResults { diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 902a1854f90..a811459d6ff 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -13,6 +13,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" + "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" @@ -95,7 +97,7 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat } // GetRestAPIsOutput function gets results from apigw api. -// GetRestApis Apigateway API is used to retrieve the specified info. This returns a map with the names and ids of RestAPIs configured +// GetRestApis Apigateway API is used to retrieve only the REST API specified info. This returns a map with the names and ids of RestAPIs configured // Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. func GetRestAPIsOutput(svcRestApi *apigateway.Client, limit *int32) (map[string]string, error) { input := &apigateway.GetRestApisInput{} @@ -111,10 +113,31 @@ func GetRestAPIsOutput(svcRestApi *apigateway.Client, limit *int32) (map[string] return nil, fmt.Errorf("error retrieving GetRestApis %w", err) } + // Iterate and display the APIs + infoRestAPImap := make(map[string]string, len(result.Items)) + for _, api := range result.Items { + infoRestAPImap[aws.StringValue(api.Name)] = aws.StringValue(api.Id) + } + return infoRestAPImap, nil +} + +// GetAPIsOutput function gets results from apigatewayv2 api. +// GetApis Apigateway API is used to retrieve the HTTP and WEBSOCKET specified info. This returns a map with the names and ids of relevant APIs configured +func GetAPIsOutput(svcHttpApi *apigatewayv2.Client) (map[string]string, error) { + input := &apigatewayv2.GetApisInput{} + + ctx, cancel := getContextWithTimeout(DefaultApiTimeout) + defer cancel() + result, err := svcHttpApi.GetApis(ctx, input) + + if err != nil { + return nil, fmt.Errorf("error retrieving GetApis %w", err) + } + // Iterate and display the APIs infoAPImap := make(map[string]string, len(result.Items)) for _, api := range result.Items { - infoAPImap[aws.StringValue(api.Name)] = aws.StringValue(api.Id) + infoAPImap[aws.StringValue(api.Name)] = aws.StringValue(api.ApiId) } return infoAPImap, nil } From 8ff8425ec802738b7593833406cb5fa4c93a4dc6 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 17:29:52 +0300 Subject: [PATCH 10/34] merging the http apigw --- .../module/aws/cloudwatch/cloudwatch.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 3236d93c72b..e5ca075ba90 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -6,6 +6,7 @@ package cloudwatch import ( "fmt" + "maps" "reflect" "strconv" "strings" @@ -196,6 +197,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // get resource type filters and tags filters for each namespace resourceTypeTagFilters := constructTagsFilters(namespaceDetails) + m.logger.Infof("Collected metrics for filter %v", resourceTypeTagFilters) //Check whether namespace is APIGW checkns := "AWS/ApiGateway" @@ -210,10 +212,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err != nil { m.Logger().Errorf("could not get http and websocket apis output: %v", err) } - m.Logger().Debugf("inforestapi response: %v", inforestapi) - m.Logger().Debugf("infoapi response: %v", infoapi) - } + maps.Copy(infoapi, inforestapi) + m.Logger().Infof("infoapi response: %v", infoapi) + } eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) @@ -592,15 +594,13 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient // And tags are only store under s3BucketName in resourceTagMap. subIdentifiers := strings.Split(identifierValue, dimensionSeparator) for _, subIdentifier := range subIdentifiers { - m.logger.Infof("PASSSS Before %s", subIdentifier) - if valAPIName, ok := infoAPImap[subIdentifier]; ok { - subIdentifier = valAPIName - m.logger.Infof("PASSSS After %s from api %s", subIdentifier, valAPIName) + if len(infoAPImap) > 0 { // If infoAPImap + if valAPIName, ok := infoAPImap[subIdentifier]; ok { + subIdentifier = valAPIName + } } - m.logger.Infof("PASSSS ola mazi %s", resourceTagMap[subIdentifier]) - if _, ok := events[uniqueIdentifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize From 5369a07e4a8475975bc2dff08f7f2d71ad2fcbd1 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 18:05:05 +0300 Subject: [PATCH 11/34] added checks for collecting rest apis only in case the check fails --- .../module/aws/cloudwatch/cloudwatch.go | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index e5ca075ba90..dc99ec43a38 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -124,7 +124,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency) m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // Initialise the map that will be used in case APiGW api is configured - inforestapi := make(map[string]string) + infotherapi := make(map[string]string) infoapi := make(map[string]string) // Check statistic method in config err := m.checkStatistics() @@ -197,22 +197,28 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // get resource type filters and tags filters for each namespace resourceTypeTagFilters := constructTagsFilters(namespaceDetails) - m.logger.Infof("Collected metrics for filter %v", resourceTypeTagFilters) //Check whether namespace is APIGW checkns := "AWS/ApiGateway" + checkresource_type := "apigateway:restapis" + if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { // inforestapi inlcudes only Rest APIs - inforestapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } - // infoapi inlcudes only WebSocket and HTTP APIs - infoapi, err = aws.GetAPIsOutput(svcHttpAPI) - if err != nil { - m.Logger().Errorf("could not get http and websocket apis output: %v", err) + + for key := range resourceTypeTagFilters { + if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) != 0 { + // infoapi inlcudes only WebSocket and HTTP APIs + infotherapi, err = aws.GetAPIsOutput(svcHttpAPI) + if err != nil { + m.Logger().Errorf("could not get http and websocket apis output: %v", err) + } + } } - maps.Copy(infoapi, inforestapi) + maps.Copy(infoapi, infotherapi) m.Logger().Infof("infoapi response: %v", infoapi) } @@ -598,6 +604,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient if len(infoAPImap) > 0 { // If infoAPImap if valAPIName, ok := infoAPImap[subIdentifier]; ok { subIdentifier = valAPIName + m.logger.Infof("PASSSS We changed %s", subIdentifier) } } From 2750f390ea946d35e6346a9585e59e8a292ea677 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 18:12:11 +0300 Subject: [PATCH 12/34] added checks for collecting rest apis only in case the check fails --- .../module/aws/cloudwatch/cloudwatch.go | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index dc99ec43a38..ade4cada3fc 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -204,21 +204,28 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { // inforestapi inlcudes only Rest APIs - infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) - if err != nil { - m.Logger().Errorf("could not get rest apis output: %v", err) - } - - for key := range resourceTypeTagFilters { - if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) != 0 { - // infoapi inlcudes only WebSocket and HTTP APIs - infotherapi, err = aws.GetAPIsOutput(svcHttpAPI) - if err != nil { - m.Logger().Errorf("could not get http and websocket apis output: %v", err) + if len(resourceTypeTagFilters) == 1 { + for key := range resourceTypeTagFilters { + if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) == 0 { + infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + if err != nil { + m.Logger().Errorf("could not get rest apis output: %v", err) + } } } + } else { + infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + if err != nil { + m.Logger().Errorf("could not get rest apis output: %v", err) + } + // infoapi inlcudes only WebSocket and HTTP APIs + infotherapi, err = aws.GetAPIsOutput(svcHttpAPI) + if err != nil { + m.Logger().Errorf("could not get http and websocket apis output: %v", err) + } + maps.Copy(infoapi, infotherapi) } - maps.Copy(infoapi, infotherapi) + m.Logger().Infof("infoapi response: %v", infoapi) } From b9ad7e6f880ef1eb1c6c941bc761ce753238edac Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 18:17:05 +0300 Subject: [PATCH 13/34] added checks for collecting rest apis only in case the check fails --- .../module/aws/cloudwatch/cloudwatch.go | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index ade4cada3fc..2aa50e2d56b 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -201,17 +201,21 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { //Check whether namespace is APIGW checkns := "AWS/ApiGateway" checkresource_type := "apigateway:restapis" + useonlyrest := false + if len(resourceTypeTagFilters) == 1 { + for key := range resourceTypeTagFilters { + if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) == 0 { + useonlyrest = true + } + } + } if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { // inforestapi inlcudes only Rest APIs - if len(resourceTypeTagFilters) == 1 { - for key := range resourceTypeTagFilters { - if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) == 0 { - infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) - if err != nil { - m.Logger().Errorf("could not get rest apis output: %v", err) - } - } + if useonlyrest { + infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + if err != nil { + m.Logger().Errorf("could not get rest apis output: %v", err) } } else { infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) From ab38936cea51f6aa0d5a44063ca0cf724c05bfeb Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 12 Sep 2024 18:34:50 +0300 Subject: [PATCH 14/34] merging with main --- NOTICE.txt | 212 ++++++++++++++++++ .../module/aws/cloudwatch/cloudwatch.go | 6 +- 2 files changed, 214 insertions(+), 4 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 06c4df765d2..1f62b266123 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -6557,6 +6557,218 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/aws/aws-sdk-go-v2/service/apigatewayv2 +Version: v1.22.8 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/apigatewayv2@v1.22.8/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/cloudformation Version: v1.53.5 diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 2aa50e2d56b..77e8d575a97 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -203,7 +203,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { checkresource_type := "apigateway:restapis" useonlyrest := false if len(resourceTypeTagFilters) == 1 { - for key := range resourceTypeTagFilters { if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) == 0 { useonlyrest = true @@ -230,7 +229,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { maps.Copy(infoapi, infotherapi) } - m.Logger().Infof("infoapi response: %v", infoapi) + m.Logger().Debugf("infoapi response: %v", infoapi) } eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) @@ -612,10 +611,9 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient subIdentifiers := strings.Split(identifierValue, dimensionSeparator) for _, subIdentifier := range subIdentifiers { - if len(infoAPImap) > 0 { // If infoAPImap + if len(infoAPImap) > 0 { // If infoAPImap includes data if valAPIName, ok := infoAPImap[subIdentifier]; ok { subIdentifier = valAPIName - m.logger.Infof("PASSSS We changed %s", subIdentifier) } } From 98918efb4cfdb27fdfe68c96bb7489e7d23b508f Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Fri, 13 Sep 2024 11:09:46 +0300 Subject: [PATCH 15/34] Update x-pack/metricbeat/module/aws/utils.go Co-authored-by: kaiyan-sheng --- x-pack/metricbeat/module/aws/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index a811459d6ff..c76580ddd30 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -99,7 +99,7 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat // GetRestAPIsOutput function gets results from apigw api. // GetRestApis Apigateway API is used to retrieve only the REST API specified info. This returns a map with the names and ids of RestAPIs configured // Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. -func GetRestAPIsOutput(svcRestApi *apigateway.Client, limit *int32) (map[string]string, error) { +func GetAPIGatewayAPIOutput(svcRestApi *apigateway.Client, limit *int32) (map[string]string, error) { input := &apigateway.GetRestApisInput{} if limit != nil { input = &apigateway.GetRestApisInput{ From 2128d2b992da83da00be4a14d7df2ce163bf170b Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 13 Sep 2024 11:25:40 +0300 Subject: [PATCH 16/34] merging with main --- .../module/aws/cloudwatch/cloudwatch.go | 15 +++++++-------- x-pack/metricbeat/module/aws/utils.go | 8 ++++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 77e8d575a97..1c768a47417 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -123,8 +123,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency) m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) - // Initialise the map that will be used in case APiGW api is configured - infotherapi := make(map[string]string) + // Initialise the map that will be used in case APIGateway api is configured. Infoapi includes Name_of_API:ID_of_API entries infoapi := make(map[string]string) // Check statistic method in config err := m.checkStatistics() @@ -210,26 +209,26 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } } if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { - // inforestapi inlcudes only Rest APIs + // inforestapi includes only Rest APIs if useonlyrest { - infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + infoapi, err = aws.GetAPIGatewayRestAPIOutput(svcRestAPI, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } } else { - infoapi, err = aws.GetRestAPIsOutput(svcRestAPI, config.LimitRestAPI) + infoapi, err = aws.GetAPIGatewayRestAPIOutput(svcRestAPI, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } - // infoapi inlcudes only WebSocket and HTTP APIs - infotherapi, err = aws.GetAPIsOutput(svcHttpAPI) + // infoapi includes only WebSocket and HTTP APIs + infotherapi, err := aws.GetAPIGatewayAPIOutput(svcHttpAPI) if err != nil { m.Logger().Errorf("could not get http and websocket apis output: %v", err) } maps.Copy(infoapi, infotherapi) } - m.Logger().Debugf("infoapi response: %v", infoapi) + m.Logger().Infof("infoapi response: %v", infoapi) } eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index c76580ddd30..49e4588bf49 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -96,10 +96,10 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat return metricWithAccountID, nil } -// GetRestAPIsOutput function gets results from apigw api. +// GetAPIGatewayRestAPIOutput function gets results from apigw api. // GetRestApis Apigateway API is used to retrieve only the REST API specified info. This returns a map with the names and ids of RestAPIs configured // Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. -func GetAPIGatewayAPIOutput(svcRestApi *apigateway.Client, limit *int32) (map[string]string, error) { +func GetAPIGatewayRestAPIOutput(svcRestApi *apigateway.Client, limit *int32) (map[string]string, error) { input := &apigateway.GetRestApisInput{} if limit != nil { input = &apigateway.GetRestApisInput{ @@ -121,9 +121,9 @@ func GetAPIGatewayAPIOutput(svcRestApi *apigateway.Client, limit *int32) (map[st return infoRestAPImap, nil } -// GetAPIsOutput function gets results from apigatewayv2 api. +// GetAPIGatewayAPIOutput function gets results from apigatewayv2 api. // GetApis Apigateway API is used to retrieve the HTTP and WEBSOCKET specified info. This returns a map with the names and ids of relevant APIs configured -func GetAPIsOutput(svcHttpApi *apigatewayv2.Client) (map[string]string, error) { +func GetAPIGatewayAPIOutput(svcHttpApi *apigatewayv2.Client) (map[string]string, error) { input := &apigatewayv2.GetApisInput{} ctx, cancel := getContextWithTimeout(DefaultApiTimeout) From 5ef7bd12ddf4fc848323eeca83f1fbb6bfc2eec4 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 13 Sep 2024 11:44:25 +0300 Subject: [PATCH 17/34] merging with main --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 1c768a47417..e8fc74e1826 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -225,7 +225,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err != nil { m.Logger().Errorf("could not get http and websocket apis output: %v", err) } - maps.Copy(infoapi, infotherapi) + if len(infotherapi) > 0 { + maps.Copy(infoapi, infotherapi) + } } m.Logger().Infof("infoapi response: %v", infoapi) From d0b2f07e67664af022b1ed2e927edec31153240a Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 13 Sep 2024 12:07:55 +0300 Subject: [PATCH 18/34] lint error --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index e8fc74e1826..3ca5ec1463c 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -19,6 +19,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" resourcegroupstaggingapitypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" From 94869b53ce164e70915eadff5a041853474f95d7 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 16 Sep 2024 16:50:27 +0300 Subject: [PATCH 19/34] adding const and creating a struct to return values --- .../module/aws/cloudwatch/cloudwatch.go | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 3ca5ec1463c..d0c161bf470 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -33,6 +33,16 @@ var ( dimensionValueWildcard = "*" ) +const checkns = "AWS/ApiGateway" +const checkresource_type = "apigateway:restapis" + +type APIClients struct { + CloudWatchClient *cloudwatch.Client + Resourcegroupstaggingapi *resourcegroupstaggingapi.Client + Apigateway *apigateway.Client + Apigatewayv2 *apigatewayv2.Client +} + // init registers the MetricSet with the central registry as soon as the program // starts. The New function will be called later to instantiate an instance of // the MetricSet for each host defined in the module's configuration. After the @@ -149,12 +159,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.logger.Debugf("Collecting metrics from AWS region %s", regionName) beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - svcCloudwatch, svcResourceAPI, _, _, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + APIClients, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName) } - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime) + eventsWithIdentifier, err := m.createEvents(APIClients.CloudWatchClient, APIClients.Resourcegroupstaggingapi, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } @@ -174,14 +184,14 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { beatsConfig := m.MetricSet.AwsConfig.Copy() beatsConfig.Region = regionName - svcCloudwatch, svcResourceAPI, svcRestAPI, svcHttpAPI, err := m.createAwsRequiredClients(beatsConfig, regionName, config) + APIClients, err := m.createAwsRequiredClients(beatsConfig, regionName, config) if err != nil { m.Logger().Warn("skipping metrics list from region '%s'", regionName, err) continue } // retrieve all the details for all the metrics available in the current region - listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, svcCloudwatch) + listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, APIClients.CloudWatchClient) if err != nil { m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err) } @@ -199,8 +209,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { resourceTypeTagFilters := constructTagsFilters(namespaceDetails) //Check whether namespace is APIGW - checkns := "AWS/ApiGateway" - checkresource_type := "apigateway:restapis" + useonlyrest := false if len(resourceTypeTagFilters) == 1 { for key := range resourceTypeTagFilters { @@ -212,17 +221,17 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { // inforestapi includes only Rest APIs if useonlyrest { - infoapi, err = aws.GetAPIGatewayRestAPIOutput(svcRestAPI, config.LimitRestAPI) + infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } } else { - infoapi, err = aws.GetAPIGatewayRestAPIOutput(svcRestAPI, config.LimitRestAPI) + infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) if err != nil { m.Logger().Errorf("could not get rest apis output: %v", err) } // infoapi includes only WebSocket and HTTP APIs - infotherapi, err := aws.GetAPIGatewayAPIOutput(svcHttpAPI) + infotherapi, err := aws.GetAPIGatewayAPIOutput(APIClients.Apigatewayv2) if err != nil { m.Logger().Errorf("could not get http and websocket apis output: %v", err) } @@ -234,7 +243,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.Logger().Infof("infoapi response: %v", infoapi) } - eventsWithIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) + eventsWithIdentifier, err := m.createEvents(APIClients.CloudWatchClient, APIClients.Resourcegroupstaggingapi, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime) if err != nil { return fmt.Errorf("createEvents failed for region %s: %w", regionName, err) } @@ -257,31 +266,32 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // createAwsRequiredClients will return the two necessary client instances to do Metric requests to the AWS API -func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, *apigateway.Client, *apigatewayv2.Client, error) { +func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (APIClients, error) { m.logger.Debugf("Collecting metrics from AWS region %s", regionName) - svcCloudwatchClient := cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) { + APIClients := APIClients{} + APIClients.CloudWatchClient = cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) { if config.AWSConfig.FIPSEnabled { o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled } }) - svcResourceAPIClient := resourcegroupstaggingapi.NewFromConfig(beatsConfig, func(o *resourcegroupstaggingapi.Options) { + APIClients.Resourcegroupstaggingapi = resourcegroupstaggingapi.NewFromConfig(beatsConfig, func(o *resourcegroupstaggingapi.Options) { if config.AWSConfig.FIPSEnabled { o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled } }) - svcGetRestAPIClient := apigateway.NewFromConfig(beatsConfig, func(o *apigateway.Options) { + APIClients.Apigateway = apigateway.NewFromConfig(beatsConfig, func(o *apigateway.Options) { }) - svcGetHttpAPIClient := apigatewayv2.NewFromConfig(beatsConfig, func(o *apigatewayv2.Options) { + APIClients.Apigatewayv2 = apigatewayv2.NewFromConfig(beatsConfig, func(o *apigatewayv2.Options) { }) - return svcCloudwatchClient, svcResourceAPIClient, svcGetRestAPIClient, svcGetHttpAPIClient, nil + return APIClients, nil } // filterListMetricsOutput compares config details with listMetricsOutput and filter out the ones don't match From 7bd732430804135067685728fbf0e669f4abd278 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 18 Sep 2024 11:26:53 +0300 Subject: [PATCH 20/34] adding vars from const --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index fae4739748a..7ce6793dd6c 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -209,16 +209,19 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { resourceTypeTagFilters := constructTagsFilters(namespaceDetails) //Check whether namespace is APIGW - + var ( + checkns_lower = strings.ToLower(checkns) + checkresource_type_lower = strings.ToLower(checkresource_type) + ) useonlyrest := false if len(resourceTypeTagFilters) == 1 { for key := range resourceTypeTagFilters { - if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) == 0 { + if strings.Compare(strings.ToLower(key), checkresource_type_lower) == 0 { useonlyrest = true } } } - if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { + if strings.Contains(strings.ToLower(namespace), checkns_lower) { // inforestapi includes only Rest APIs if useonlyrest { infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) From 38c8cae4c0c24048d77357b969b968574a0b9949 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 18 Sep 2024 11:26:53 +0300 Subject: [PATCH 21/34] adding vars from const fixing chagelog deleting unneeded manifest Changes to be committed: deleted: metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat2.yaml --- CHANGELOG.next.asciidoc | 1 - x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c685eccbc4c..20cd2f6589d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -332,7 +332,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add new metricset network for the vSphere module. {pull}40559[40559] - Add new metricset resourcepool for the vSphere module. {pull}40456[40456] - Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway resources {pull}40755[40755] -- Add new metricset datastorecluster for vSphere module. {pull}40634[40634] {pull}40694[40694] - Add new metricset datastorecluster for vSphere module. {pull}40634[40634] - Add support for new metrics in datastorecluster metricset. {pull}40694[40694] - Add new metrics for the vSphere Virtualmachine metricset. {pull}40485[40485] diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index fae4739748a..7ce6793dd6c 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -209,16 +209,19 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { resourceTypeTagFilters := constructTagsFilters(namespaceDetails) //Check whether namespace is APIGW - + var ( + checkns_lower = strings.ToLower(checkns) + checkresource_type_lower = strings.ToLower(checkresource_type) + ) useonlyrest := false if len(resourceTypeTagFilters) == 1 { for key := range resourceTypeTagFilters { - if strings.Compare(strings.ToLower(key), strings.ToLower(checkresource_type)) == 0 { + if strings.Compare(strings.ToLower(key), checkresource_type_lower) == 0 { useonlyrest = true } } } - if strings.Contains(strings.ToLower(namespace), strings.ToLower(checkns)) { + if strings.Contains(strings.ToLower(namespace), checkns_lower) { // inforestapi includes only Rest APIs if useonlyrest { infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) From 0acaf513ff6ef3bb782cddf2a7f7229443a671ac Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 19 Sep 2024 14:20:21 +0300 Subject: [PATCH 22/34] updating docs --- metricbeat/docs/modules/aws.asciidoc | 19 +++++++++++++++++++ .../metricbeat/module/aws/_meta/docs.asciidoc | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 20b6854a834..75379e7cfff 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -146,6 +146,25 @@ Enforces the use of FIPS service endpoints. See < Date: Sat, 21 Sep 2024 13:57:45 +0300 Subject: [PATCH 23/34] Update x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go Co-authored-by: kaiyan-sheng --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index f718ce19046..f478c2fc21c 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1261,7 +1261,9 @@ func (m *MockResourceGroupsTaggingClient) GetResources(context.Context, *resourc type MockResourceGroupsTaggingClient2 struct{} // GetResources implements resourcegroupstaggingapi.GetResourcesAPIClient. -func (m *MockResourceGroupsTaggingClient2) GetResources(context.Context, *apigateway.GetResourcesInput, ...func(*apigateway.Options)) (*restapi.GetResourcesOutput, error) { +func (m *MockResourceGroupsTaggingClient2) GetResources(context.Context, *apigateway.GetResourcesInput, ...func(*apigateway.Options)) (*apigateway.GetResourcesOutput, error) { + return &apigateway.GetResourcesOutput{}, nil +} return &restapi.GetResourcesOutput{}, nil } From 03bd34af6743d20fd0faa2be1d5e2516e0541ecc Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Sat, 21 Sep 2024 13:58:27 +0300 Subject: [PATCH 24/34] Update x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go Co-authored-by: kaiyan-sheng --- .../module/aws/cloudwatch/cloudwatch.go | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 7ce6793dd6c..38ad0efe2a0 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -234,7 +234,26 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.Logger().Errorf("could not get rest apis output: %v", err) } // infoapi includes only WebSocket and HTTP APIs - infotherapi, err := aws.GetAPIGatewayAPIOutput(APIClients.Apigatewayv2) + // apiGatewayRestAPI includes only Rest APIs + if useonlyrest { + apiGatewayRestAPI, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) + if err != nil { + m.Logger().Errorf("could not get rest apis output: %v", err) + } + } else { + apiGatewayRestAPI, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) + if err != nil { + m.Logger().Errorf("could not get rest apis output: %v", err) + } + // apiGatewayAPI includes only WebSocket and HTTP APIs + apiGatewayAPI, err := aws.GetAPIGatewayAPIOutput(APIClients.Apigatewayv2) + if err != nil { + m.Logger().Errorf("could not get http and websocket apis output: %v", err) + } + if len(apiGatewayAPI) > 0 { + maps.Copy(apiGatewayRestAPI, apiGatewayAPI) + } + } if err != nil { m.Logger().Errorf("could not get http and websocket apis output: %v", err) } From 26a4375ba45c4cb8f35480c75cf8e6afb342637c Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Mon, 23 Sep 2024 13:21:56 +0300 Subject: [PATCH 25/34] adding a validation for the LimitRestAPI --- x-pack/metricbeat/module/aws/aws.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index a0c92d663ae..1607a60c52c 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -208,6 +208,18 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { }) metricSet.MonitoringAccountName = getAccountName(svcIam, base, metricSet) + //Validate LimitRestAPI value. + //The Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. + if config.LimitRestAPI != nil { + if *config.LimitRestAPI > 500 { + base.Logger().Debug("Metricset LimitRestAPI config value can not exceed value 500. Setting LimitRestAPI=25") + *config.LimitRestAPI = 25 + } else if *config.LimitRestAPI <= 0 { + base.Logger().Debug("Metricset LimitRestAPI config value can not be <=0. Setting LimitRestAPI=25") + *config.LimitRestAPI = 25 + } + } + // Construct MetricSet with a full regions list if config.Regions == nil { svcEC2 := ec2.NewFromConfig(awsConfig, func(o *ec2.Options) { From 705ffb56df763c8a5059e6284747e7fffb3d81c7 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 27 Sep 2024 18:09:15 +0300 Subject: [PATCH 26/34] merging with main --- x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 2 +- x-pack/metricbeat/module/aws/aws.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index bbaa06c2bb8..6d5d0240d74 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -150,7 +150,7 @@ See https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/apigateway@v1.25.8#G metrics: - namespace: "AWS/ApiGateway" resource_type: "apigateway:restapis" - limit_rest_api: 40 + max_apigateway_results: 40 ---- The aws module comes with a predefined dashboard. For example: diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 1607a60c52c..e024ecb5ef4 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -36,7 +36,7 @@ type Config struct { AWSConfig awscommon.ConfigAWS `config:",inline"` TagsFilter []Tag `config:"tags_filter"` IncludeLinkedAccounts *bool `config:"include_linked_accounts"` - LimitRestAPI *int32 `config:"limit_rest_api"` + LimitRestAPI *int32 `config:"max_apigateway_results"` OwningAccount string `config:"owning_account"` } From c1e2cf23da2cd40834292a10088dd3f424a96b38 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Fri, 27 Sep 2024 18:27:25 +0300 Subject: [PATCH 27/34] merging with main --- metricbeat/docs/modules/aws.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 75379e7cfff..942c0519677 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -162,7 +162,7 @@ See https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/apigateway@v1.25.8#G metrics: - namespace: "AWS/ApiGateway" resource_type: "apigateway:restapis" - limit_rest_api: 40 + max_apigateway_results: 40 ---- The aws module comes with a predefined dashboard. For example: From d1894cdc6957916ee07ff8f6d5f2bff28c10b015 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 2 Oct 2024 11:06:34 +0300 Subject: [PATCH 28/34] fixing docs --- NOTICE.txt | 4 ++-- x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 2 +- x-pack/metricbeat/module/aws/aws.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 470f2c19810..00217700d5b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5063,11 +5063,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-lambda-go@v1.44 -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go -Version: v1.38.60 +Version: v1.54.19 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go@v1.38.60/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go@v1.54.19/LICENSE.txt: Apache License diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 6d5d0240d74..efe152e54f5 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -134,7 +134,7 @@ Enforces the use of FIPS service endpoints. See < 500 { - base.Logger().Debug("Metricset LimitRestAPI config value can not exceed value 500. Setting LimitRestAPI=25") + base.Logger().Info("max_apigateway_results config value can not exceed value 500. Setting max_apigateway_results=25") *config.LimitRestAPI = 25 } else if *config.LimitRestAPI <= 0 { - base.Logger().Debug("Metricset LimitRestAPI config value can not be <=0. Setting LimitRestAPI=25") + base.Logger().Info("max_apigateway_results config value can not be <=0. Setting max_apigateway_results=25") *config.LimitRestAPI = 25 } } From dce6fd28436ff420fedb5ff1b03c704186d7462e Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 2 Oct 2024 11:13:14 +0300 Subject: [PATCH 29/34] fixing docs --- metricbeat/docs/modules/aws.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 942c0519677..c964607c83a 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -146,7 +146,7 @@ Enforces the use of FIPS service endpoints. See < Date: Wed, 2 Oct 2024 15:09:07 +0300 Subject: [PATCH 30/34] fixing docs --- metricbeat/docs/modules/aws.asciidoc | 4 ++-- x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 4 ++-- x-pack/metricbeat/module/aws/aws.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index c964607c83a..8df1f3cde13 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -146,7 +146,7 @@ Enforces the use of FIPS service endpoints. See < 500 { - base.Logger().Info("max_apigateway_results config value can not exceed value 500. Setting max_apigateway_results=25") + base.Logger().Info("max_apigateway_results config value can not exceed value 500. Setting apigateway_max_results=25") *config.LimitRestAPI = 25 } else if *config.LimitRestAPI <= 0 { - base.Logger().Info("max_apigateway_results config value can not be <=0. Setting max_apigateway_results=25") + base.Logger().Info("max_apigateway_results config value can not be <=0. Setting apigateway_max_results=25") *config.LimitRestAPI = 25 } } From d65b4092cce9577fae48e5a9c0c89eca09852a42 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 2 Oct 2024 15:24:27 +0300 Subject: [PATCH 31/34] variables set to apigateway_max_results --- x-pack/metricbeat/module/aws/aws.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f5e24f79dca..12066bc3936 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -212,10 +212,10 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { //The Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. if config.LimitRestAPI != nil { if *config.LimitRestAPI > 500 { - base.Logger().Info("max_apigateway_results config value can not exceed value 500. Setting apigateway_max_results=25") + base.Logger().Debug("apigateway_max_results config value can not exceed value 500. Setting apigateway_max_results=25") *config.LimitRestAPI = 25 } else if *config.LimitRestAPI <= 0 { - base.Logger().Info("max_apigateway_results config value can not be <=0. Setting apigateway_max_results=25") + base.Logger().Debug("apigateway_max_results config value can not be <=0. Setting apigateway_max_results=25") *config.LimitRestAPI = 25 } } From c5139b38514a4573a79e17867aeb7b493e0bc7a3 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Wed, 2 Oct 2024 15:37:44 +0300 Subject: [PATCH 32/34] moving check inside apigw namespace --- .../metricbeat/module/aws/cloudwatch/cloudwatch.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index ad3d9865d9d..bb6e6f7e39a 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -213,15 +213,15 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { resourceTypeTagFilters := constructTagsFilters(namespaceDetails) //Check whether namespace is APIGW - useonlyrest := false - if len(resourceTypeTagFilters) == 1 { - for key := range resourceTypeTagFilters { - if strings.Compare(strings.ToLower(key), checkresource_type_lower) == 0 { - useonlyrest = true + if strings.Contains(strings.ToLower(namespace), checkns_lower) { + useonlyrest := false + if len(resourceTypeTagFilters) == 1 { + for key := range resourceTypeTagFilters { + if strings.Compare(strings.ToLower(key), checkresource_type_lower) == 0 { + useonlyrest = true + } } } - } - if strings.Contains(strings.ToLower(namespace), checkns_lower) { // inforestapi includes only Rest APIs if useonlyrest { infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI) From f95c8e42133976e6a87046ac521730763e4c2f5f Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 3 Oct 2024 08:47:11 +0300 Subject: [PATCH 33/34] setting to max limit to 500 --- x-pack/metricbeat/module/aws/aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 12066bc3936..fc3962f6b65 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -213,7 +213,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { if config.LimitRestAPI != nil { if *config.LimitRestAPI > 500 { base.Logger().Debug("apigateway_max_results config value can not exceed value 500. Setting apigateway_max_results=25") - *config.LimitRestAPI = 25 + *config.LimitRestAPI = 500 } else if *config.LimitRestAPI <= 0 { base.Logger().Debug("apigateway_max_results config value can not be <=0. Setting apigateway_max_results=25") *config.LimitRestAPI = 25 From e2376ea5e29b64dffaba0fce0dc1f40a6aa84f52 Mon Sep 17 00:00:00 2001 From: Andreas Gkizas Date: Thu, 3 Oct 2024 09:07:15 +0300 Subject: [PATCH 34/34] setting to max limit to 500 --- x-pack/metricbeat/module/aws/aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index fc3962f6b65..0b8a559fc31 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -212,7 +212,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { //The Limit variable defines maximum number of returned results per page. The default value is 25 and the maximum value is 500. if config.LimitRestAPI != nil { if *config.LimitRestAPI > 500 { - base.Logger().Debug("apigateway_max_results config value can not exceed value 500. Setting apigateway_max_results=25") + base.Logger().Debug("apigateway_max_results config value can not exceed value 500. Setting apigateway_max_results=500") *config.LimitRestAPI = 500 } else if *config.LimitRestAPI <= 0 { base.Logger().Debug("apigateway_max_results config value can not be <=0. Setting apigateway_max_results=25")