From f8638dbc5bdb7e67b3a037120780d7a13036e0d7 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 22 Nov 2022 15:19:56 +0100 Subject: [PATCH 1/9] update elastic-agent-autodiscover; use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources Signed-off-by: Tetiana Kravchenko --- go.mod | 2 +- go.sum | 6 ++++-- metricbeat/module/kubernetes/util/kubernetes.go | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 425b2d777d3..c30da360635 100644 --- a/go.mod +++ b/go.mod @@ -192,7 +192,7 @@ require ( github.com/aws/smithy-go v1.12.0 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.4.0 + github.com/elastic/elastic-agent-autodiscover v0.5.0 github.com/elastic/elastic-agent-libs v0.2.13 github.com/elastic/elastic-agent-shipper-client v0.4.0 github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51 diff --git a/go.sum b/go.sum index df7214bef16..8b06b7d6da1 100644 --- a/go.sum +++ b/go.sum @@ -617,8 +617,10 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk= github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe h1:fpM0Lw4JTA9XYbQMQhVrMzJ0GrwUaXHVSsrm5n/b23E= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8= +github.com/elastic/elastic-agent-autodiscover v0.5.0 h1:fiDJQKB148RsRISqqbGINtJ/ZPupeVjwn0fbPz9ZDOc= +github.com/elastic/elastic-agent-autodiscover v0.5.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd h1:IuAuac3vcucBrjAXKPQlTJ22H7mBUsSnNWxa7GZYFEg= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.13 h1:YQzhO8RaLosGlyt7IHtj/ZxigWiwLcXXlv3gS4QY9CA= github.com/elastic/elastic-agent-libs v0.2.13/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss= diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index e6addbba791..2ba4dbf4042 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -105,11 +105,13 @@ func NewResourceMetadataEnricher( } cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.NewResourceMetadataGenerator(cfg, watcher.Client()) podMetaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, config.AddResourceMetadata) namespaceMeta := metadata.NewNamespaceMetadataGenerator(config.AddResourceMetadata.Namespace, namespaceWatcher.Store(), watcher.Client()) serviceMetaGen := metadata.NewServiceMetadataGenerator(cfg, watcher.Store(), namespaceMeta, watcher.Client()) + + metaGen := metadata.NewNamespaceAwareResourceMetadataGenerator(cfg, watcher.Client(), namespaceMeta) + enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, // update func(m map[string]mapstr.M, r kubernetes.Resource) { From f4ace20a21ee3890c341395cdc6adf5937081fc2 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 22 Nov 2022 17:26:51 +0100 Subject: [PATCH 2/9] update configuration comments Signed-off-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/kubernetes.asciidoc | 6 ++---- metricbeat/metricbeat.reference.yml | 6 ++---- metricbeat/module/kubernetes/_meta/config.reference.yml | 6 ++---- metricbeat/module/kubernetes/_meta/config.yml | 3 +-- metricbeat/modules.d/kubernetes.yml.disabled | 3 +-- x-pack/metricbeat/metricbeat.reference.yml | 6 ++---- 7 files changed, 11 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5b40f808c12..f84f8aa300a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -186,6 +186,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308] - Add per-thread metrics to system_summary {pull}33614[33614] - Add GCP CloudSQL metadata {pull}33066[33066] +- Add namespace metadata to all namespaced kubernetes resources. {pull} *Packetbeat* diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 880a17ef3a6..fa11e3b5a06 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -237,8 +237,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -284,8 +283,7 @@ metricbeat.modules: #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index acd1f4459ad..21be41101b4 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -506,8 +506,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -553,8 +552,7 @@ metricbeat.modules: #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index f95800a0a64..dcd59309119 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -23,8 +23,7 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -70,8 +69,7 @@ #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index e29da38718f..44ef19c9786 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -25,8 +25,7 @@ #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 307329accd6..23bd210a835 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -28,8 +28,7 @@ #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 143ddccf9ce..e6931fbb309 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -888,8 +888,7 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: @@ -935,8 +934,7 @@ metricbeat.modules: #kube_config: ~/.kube/config # Set the namespace to watch for resources #namespace: staging - # To configure additionally node and namespace metadata, added to pod, service and container resource types, - # `add_resource_metadata` can be defined. + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: # namespace: From 25b3e4ce173768f710b7d7a3461f6592609eec3d Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 22 Nov 2022 17:30:55 +0100 Subject: [PATCH 3/9] add PR number; fix conflicts Signed-off-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f84f8aa300a..1790e1953e7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -187,6 +187,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add per-thread metrics to system_summary {pull}33614[33614] - Add GCP CloudSQL metadata {pull}33066[33066] - Add namespace metadata to all namespaced kubernetes resources. {pull} +- Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763] *Packetbeat* From 841fb4b07afba51cc7cb4a4e1cd74184e100b6d9 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 22 Nov 2022 17:37:06 +0100 Subject: [PATCH 4/9] run make notice Signed-off-by: Tetiana Kravchenko --- NOTICE.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 2f7263410c7..f1ac7a6f5e0 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9656,11 +9656,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.4.0 +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.4.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.5.0/LICENSE: Apache License Version 2.0, January 2004 From 622ec2db1d088886b934ec116787fe979201a6e5 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 23 Nov 2022 09:48:32 +0100 Subject: [PATCH 5/9] fix go.sum file after rebase Signed-off-by: Tetiana Kravchenko --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 8b06b7d6da1..69b14d6b0a8 100644 --- a/go.sum +++ b/go.sum @@ -619,8 +619,8 @@ github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-autodiscover v0.5.0 h1:fiDJQKB148RsRISqqbGINtJ/ZPupeVjwn0fbPz9ZDOc= github.com/elastic/elastic-agent-autodiscover v0.5.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd h1:IuAuac3vcucBrjAXKPQlTJ22H7mBUsSnNWxa7GZYFEg= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe h1:fpM0Lw4JTA9XYbQMQhVrMzJ0GrwUaXHVSsrm5n/b23E= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.13 h1:YQzhO8RaLosGlyt7IHtj/ZxigWiwLcXXlv3gS4QY9CA= github.com/elastic/elastic-agent-libs v0.2.13/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss= From d115383d023cbef3bab7307f672b8a77d36dc0f2 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 23 Nov 2022 11:22:53 +0100 Subject: [PATCH 6/9] make update Signed-off-by: Tetiana Kravchenko --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 69b14d6b0a8..9364ccc736b 100644 --- a/go.sum +++ b/go.sum @@ -615,8 +615,6 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk= -github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-autodiscover v0.5.0 h1:fiDJQKB148RsRISqqbGINtJ/ZPupeVjwn0fbPz9ZDOc= github.com/elastic/elastic-agent-autodiscover v0.5.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe h1:fpM0Lw4JTA9XYbQMQhVrMzJ0GrwUaXHVSsrm5n/b23E= From 1353babed5e5f9d6442dbcb8d953eac6cacf640b Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 23 Nov 2022 11:24:19 +0100 Subject: [PATCH 7/9] clean up changelog Signed-off-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1790e1953e7..81faaad3fdd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -186,7 +186,6 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308] - Add per-thread metrics to system_summary {pull}33614[33614] - Add GCP CloudSQL metadata {pull}33066[33066] -- Add namespace metadata to all namespaced kubernetes resources. {pull} - Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763] *Packetbeat* From a0234fe180baf7a5a1f9bdb53874417105ec19e8 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 28 Nov 2022 15:38:17 +0100 Subject: [PATCH 8/9] test llc Signed-off-by: Tetiana Kravchenko --- metricbeat/module/kubernetes/util/kubernetes.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 2ba4dbf4042..d8ee30f507e 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -366,10 +366,10 @@ func GetValidatedConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { } config, err = validateConfig(config) - if err != nil { - logp.Err("Error while validating config: %v", err) - return nil, err - } + // if err != nil { + // logp.Err("Error while validating config: %v", err) + // return nil, err + // } return config, nil } From 2db057a7eab7d157a924d193d68319cbe5fe59df Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 28 Nov 2022 21:33:03 +0100 Subject: [PATCH 9/9] revert local test Signed-off-by: Tetiana Kravchenko --- metricbeat/module/kubernetes/util/kubernetes.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index d8ee30f507e..2ba4dbf4042 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -366,10 +366,10 @@ func GetValidatedConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { } config, err = validateConfig(config) - // if err != nil { - // logp.Err("Error while validating config: %v", err) - // return nil, err - // } + if err != nil { + logp.Err("Error while validating config: %v", err) + return nil, err + } return config, nil }