From 951b244e0e1365972d736fda96c05ded909efc18 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 7 Dec 2022 14:10:34 +0100 Subject: [PATCH] Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763) * update elastic-agent-autodiscover; use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources Signed-off-by: Tetiana Kravchenko * update configuration comments Signed-off-by: Tetiana Kravchenko * add PR number; fix conflicts Signed-off-by: Tetiana Kravchenko * run make notice Signed-off-by: Tetiana Kravchenko * fix go.sum file after rebase Signed-off-by: Tetiana Kravchenko * make update Signed-off-by: Tetiana Kravchenko * clean up changelog Signed-off-by: Tetiana Kravchenko * test llc Signed-off-by: Tetiana Kravchenko * revert local test Signed-off-by: Tetiana Kravchenko Signed-off-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- metricbeat/docs/modules/kubernetes.asciidoc | 6 ++---- metricbeat/metricbeat.reference.yml | 6 ++---- metricbeat/module/kubernetes/_meta/config.reference.yml | 6 ++---- metricbeat/module/kubernetes/_meta/config.yml | 3 +-- metricbeat/module/kubernetes/util/kubernetes.go | 4 +++- metricbeat/modules.d/kubernetes.yml.disabled | 3 +-- x-pack/metricbeat/metricbeat.reference.yml | 6 ++---- 11 files changed, 19 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 03739d26429..ef463f559cc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -191,6 +191,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Remove GCP Compute metadata cache {pull}33655[33655] - Add support for multiple regions in GCP {pull}32964[32964] - Add GCP Redis regions support {pull}33728[33728] +- Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 054c943e2d2..ab5c1b20129 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9656,11 +9656,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.4.0 +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.4.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.5.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index e72d4a1bda3..e9405c2e4b9 100644 --- a/go.mod +++ b/go.mod @@ -192,7 +192,7 @@ require ( github.com/aws/smithy-go v1.12.0 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.4.0 + github.com/elastic/elastic-agent-autodiscover v0.5.0 github.com/elastic/elastic-agent-libs v0.2.15 github.com/elastic/elastic-agent-shipper-client v0.4.0 github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51 diff --git a/go.sum b/go.sum index 217f902e787..447cf675a89 100644 --- a/go.sum +++ b/go.sum @@ -615,8 +615,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk= -github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= +github.com/elastic/elastic-agent-autodiscover v0.5.0 h1:fiDJQKB148RsRISqqbGINtJ/ZPupeVjwn0fbPz9ZDOc= +github.com/elastic/elastic-agent-autodiscover v0.5.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-client/v7 v7.0.2-0.20221129150247-15881a8e64ef h1:+3AWaimDL826eoU06qOFBtA3xmyuTr9YUMVWvnim4mU= github.com/elastic/elastic-agent-client/v7 v7.0.2-0.20221129150247-15881a8e64ef/go.mod h1:cHviLpA5fAwMbfBIHBVNl16qp90bO7pKHMAQaG+9raU= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 880a17ef3a6..fa11e3b5a06 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -237,8 +237,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -284,8 +283,7 @@ metricbeat.modules: #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index acd1f4459ad..21be41101b4 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -506,8 +506,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -553,8 +552,7 @@ metricbeat.modules: #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index f95800a0a64..dcd59309119 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -23,8 +23,7 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -70,8 +69,7 @@ #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index e29da38718f..44ef19c9786 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -25,8 +25,7 @@ #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index e6addbba791..2ba4dbf4042 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -105,11 +105,13 @@ func NewResourceMetadataEnricher( } cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.NewResourceMetadataGenerator(cfg, watcher.Client()) podMetaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, config.AddResourceMetadata) namespaceMeta := metadata.NewNamespaceMetadataGenerator(config.AddResourceMetadata.Namespace, namespaceWatcher.Store(), watcher.Client()) serviceMetaGen := metadata.NewServiceMetadataGenerator(cfg, watcher.Store(), namespaceMeta, watcher.Client()) + + metaGen := metadata.NewNamespaceAwareResourceMetadataGenerator(cfg, watcher.Client(), namespaceMeta) + enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, // update func(m map[string]mapstr.M, r kubernetes.Resource) { diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 307329accd6..23bd210a835 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -28,8 +28,7 @@ #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 143ddccf9ce..e6931fbb309 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -888,8 +888,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -935,8 +934,7 @@ metricbeat.modules: #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: