From 9089e585e2b532bf39ed2bbb3b121e049aed46fb Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 7 Dec 2022 14:10:34 +0100 Subject: [PATCH] Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763) * update elastic-agent-autodiscover; use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources Signed-off-by: Tetiana Kravchenko * update configuration comments Signed-off-by: Tetiana Kravchenko * add PR number; fix conflicts Signed-off-by: Tetiana Kravchenko * run make notice Signed-off-by: Tetiana Kravchenko * fix go.sum file after rebase Signed-off-by: Tetiana Kravchenko * make update Signed-off-by: Tetiana Kravchenko * clean up changelog Signed-off-by: Tetiana Kravchenko * test llc Signed-off-by: Tetiana Kravchenko * revert local test Signed-off-by: Tetiana Kravchenko Signed-off-by: Tetiana Kravchenko (cherry picked from commit 951b244e0e1365972d736fda96c05ded909efc18) # Conflicts: # NOTICE.txt # go.mod # go.sum # metricbeat/docs/modules/kubernetes.asciidoc # metricbeat/metricbeat.reference.yml # metricbeat/module/kubernetes/_meta/config.reference.yml # metricbeat/module/kubernetes/_meta/config.yml # metricbeat/modules.d/kubernetes.yml.disabled # x-pack/metricbeat/metricbeat.reference.yml --- CHANGELOG.next.asciidoc | 10 ++++ NOTICE.txt | 9 +++ go.mod | 27 +++++++++ go.sum | 7 +++ metricbeat/docs/modules/kubernetes.asciidoc | 58 +++++++++++++++++++ metricbeat/metricbeat.reference.yml | 58 +++++++++++++++++++ .../kubernetes/_meta/config.reference.yml | 58 +++++++++++++++++++ metricbeat/module/kubernetes/_meta/config.yml | 6 ++ .../module/kubernetes/util/kubernetes.go | 4 +- metricbeat/modules.d/kubernetes.yml.disabled | 6 ++ x-pack/metricbeat/metricbeat.reference.yml | 58 +++++++++++++++++++ 11 files changed, 300 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2936bb71eb0..48282754792 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -81,6 +81,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Metricbeat* +- Azure Billing: upgrade Usage Details API to version 2019-10-01 {pull}31970[31970] +- Azure Billing: switch to Cost Management API for forecast data {pull}32589[32589] +- Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}33166[33166] +- Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308] +- Add per-thread metrics to system_summary {pull}33614[33614] +- Add GCP CloudSQL metadata {pull}33066[33066] +- Remove GCP Compute metadata cache {pull}33655[33655] +- Add support for multiple regions in GCP {pull}32964[32964] +- Add GCP Redis regions support {pull}33728[33728] +- Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index b57c412e716..77c29d8e9a6 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9710,6 +9710,7 @@ SOFTWARE. -------------------------------------------------------------------------------- +<<<<<<< HEAD Dependency : github.com/gocarina/gocsv Version: v0.0.0-20170324095351-ffef3ffc77be Licence type (autodetected): MIT @@ -10134,6 +10135,14 @@ Licence type (autodetected): Apache-2.0 Contents of probable licence file $GOMODCACHE/github.com/golang/mock@v1.6.0/LICENSE: +======= +Dependency : github.com/elastic/elastic-agent-autodiscover +Version: v0.5.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.5.0/LICENSE: +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 0d9bf41d48d..54cd23dc6c2 100644 --- a/go.mod +++ b/go.mod @@ -196,8 +196,35 @@ require ( ) require ( +<<<<<<< HEAD github.com/elastic/elastic-agent-libs v0.2.11 github.com/elastic/elastic-agent-system-metrics v0.4.4 +======= + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 + github.com/Azure/go-autorest/autorest/adal v0.9.14 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17 + github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4 + github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8 + github.com/aws/smithy-go v1.12.0 + github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 + github.com/elastic/bayeux v1.0.5 + github.com/elastic/elastic-agent-autodiscover v0.5.0 + github.com/elastic/elastic-agent-libs v0.2.15 + github.com/elastic/elastic-agent-shipper-client v0.4.0 + github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51 + github.com/elastic/go-elasticsearch/v8 v8.2.0 + github.com/elastic/mito v0.0.0-20221207004749-2f0f2875e464 + github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 + github.com/google/cel-go v0.12.5 + github.com/googleapis/gax-go/v2 v2.5.1 + github.com/pierrec/lz4/v4 v4.1.15 + github.com/shirou/gopsutil/v3 v3.21.12 + go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 + go.elastic.co/apm/module/apmhttp/v2 v2.0.0 + go.elastic.co/apm/v2 v2.0.0 + go.mongodb.org/mongo-driver v1.5.1 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) ) require ( diff --git a/go.sum b/go.sum index 5561ba29635..3b2d07854fe 100644 --- a/go.sum +++ b/go.sum @@ -323,11 +323,18 @@ github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible h1:jlUO91EFZu github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible/go.mod h1:4zuQekLQi489ShcqTmS1Zj1ta0qrcNBlSuGa+ziu2vM= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= +<<<<<<< HEAD github.com/elastic/ecs v1.12.0 h1:u6WZ2AWtxv5vHvTQ4EuVZdWZ51mKHQ2UIltRePcta5U= github.com/elastic/ecs v1.12.0/go.mod h1:pgiLbQsijLOJvFR8OTILLu0Ni/R/foUNg0L+T6mU9b4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= +======= +github.com/elastic/elastic-agent-autodiscover v0.5.0 h1:fiDJQKB148RsRISqqbGINtJ/ZPupeVjwn0fbPz9ZDOc= +github.com/elastic/elastic-agent-autodiscover v0.5.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= +github.com/elastic/elastic-agent-client/v7 v7.0.2-0.20221129150247-15881a8e64ef h1:+3AWaimDL826eoU06qOFBtA3xmyuTr9YUMVWvnim4mU= +github.com/elastic/elastic-agent-client/v7 v7.0.2-0.20221129150247-15881a8e64ef/go.mod h1:cHviLpA5fAwMbfBIHBVNl16qp90bO7pKHMAQaG+9raU= +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0= github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI= diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index f2b9f71c5f6..4a6c0f1cf23 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -231,6 +231,23 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # State metrics from kube-state-metrics service: - module: kubernetes @@ -261,6 +278,47 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # Set the namespace to watch for resources + #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 + +# Kubernetes Events +- module: kubernetes + enabled: true + metricsets: + - event + period: 10s + # Skip events older than Metricbeat's statup time is enabled by default. + # Setting to false the skip_older setting will stop filtering older events. + # This setting is also useful went Event's timestamps are not populated properly. + #skip_older: false + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + # Set the namespace to watch for events + #namespace: staging + # Set the sync period of the watchers + #sync_period: 10m + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # Kubernetes API server # (when running metricbeat as a deployment) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 601c1712088..05162e716f9 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -500,6 +500,23 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # State metrics from kube-state-metrics service: - module: kubernetes @@ -530,6 +547,47 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # Set the namespace to watch for resources + #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 + +# Kubernetes Events +- module: kubernetes + enabled: true + metricsets: + - event + period: 10s + # Skip events older than Metricbeat's statup time is enabled by default. + # Setting to false the skip_older setting will stop filtering older events. + # This setting is also useful went Event's timestamps are not populated properly. + #skip_older: false + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + # Set the namespace to watch for events + #namespace: staging + # Set the sync period of the watchers + #sync_period: 10m + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # Kubernetes API server # (when running metricbeat as a deployment) diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index 9b503425b9f..23cabdfc87e 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -23,6 +23,23 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # State metrics from kube-state-metrics service: - module: kubernetes @@ -53,6 +70,47 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # Set the namespace to watch for resources + #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 + +# Kubernetes Events +- module: kubernetes + enabled: true + metricsets: + - event + period: 10s + # Skip events older than Metricbeat's statup time is enabled by default. + # Setting to false the skip_older setting will stop filtering older events. + # This setting is also useful went Event's timestamps are not populated properly. + #skip_older: false + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + # Set the namespace to watch for events + #namespace: staging + # Set the sync period of the watchers + #sync_period: 10m + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # Kubernetes API server # (when running metricbeat as a deployment) diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index d728eb356e2..c5d6fc3d653 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -23,8 +23,14 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD # To configure additionally node and namespace metadata, added to pod, service and container resource types, # `add_resource_metadata` can be defined. +======= + # Set the namespace to watch for resources + #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 92fd9b7ecf5..2f393772445 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -99,10 +99,12 @@ func NewResourceMetadataEnricher( cfg, _ := common.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.NewResourceMetadataGenerator(cfg, watcher.Client()) podMetaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, config.AddResourceMetadata) namespaceMeta := metadata.NewNamespaceMetadataGenerator(config.AddResourceMetadata.Namespace, namespaceWatcher.Store(), watcher.Client()) serviceMetaGen := metadata.NewServiceMetadataGenerator(cfg, watcher.Store(), namespaceMeta, watcher.Client()) + + metaGen := metadata.NewNamespaceAwareResourceMetadataGenerator(cfg, watcher.Client(), namespaceMeta) + enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, // update func(m map[string]common.MapStr, r kubernetes.Resource) { diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index a0793347051..a9214abc52f 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -26,8 +26,14 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD # To configure additionally node and namespace metadata, added to pod, service and container resource types, # `add_resource_metadata` can be defined. +======= + # Set the namespace to watch for resources + #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 2e146e1635f..1222653cbdb 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -882,6 +882,23 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # State metrics from kube-state-metrics service: - module: kubernetes @@ -912,6 +929,47 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config +<<<<<<< HEAD +======= + # Set the namespace to watch for resources + #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. + # By default all labels will be included while annotations are not added by default. + # add_resource_metadata: + # namespace: + # include_labels: ["namespacelabel1"] + # node: + # include_labels: ["nodelabel2"] + # include_annotations: ["nodeannotation1"] + # deployment: false + # cronjob: false + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 + +# Kubernetes Events +- module: kubernetes + enabled: true + metricsets: + - event + period: 10s + # Skip events older than Metricbeat's statup time is enabled by default. + # Setting to false the skip_older setting will stop filtering older events. + # This setting is also useful went Event's timestamps are not populated properly. + #skip_older: false + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + # Set the namespace to watch for events + #namespace: staging + # Set the sync period of the watchers + #sync_period: 10m + # Kubernetes client QPS and burst can be configured additionally + #kube_client_options: + # qps: 5 + # burst: 10 +>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763)) # Kubernetes API server # (when running metricbeat as a deployment)