Use NamespaceAwareResourceMetaGenerator for all generic kubernetes re…

…sources (#33763) * update elastic-agent-autodiscover; use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources Signed-off-by: Tetiana Kravchenko <[email protected]> * update configuration comments Signed-off-by: Tetiana Kravchenko <[email protected]> * add PR number; fix conflicts Signed-off-by: Tetiana Kravchenko <[email protected]> * run make notice Signed-off-by: Tetiana Kravchenko <[email protected]> * fix go.sum file after rebase Signed-off-by: Tetiana Kravchenko <[email protected]> * make update Signed-off-by: Tetiana Kravchenko <[email protected]> * clean up changelog Signed-off-by: Tetiana Kravchenko <[email protected]> * test llc Signed-off-by: Tetiana Kravchenko <[email protected]> * revert local test Signed-off-by: Tetiana Kravchenko <[email protected]> Signed-off-by: Tetiana Kravchenko <[email protected]> (cherry picked from commit 951b244) # Conflicts: # NOTICE.txt # go.mod # go.sum # metricbeat/docs/modules/kubernetes.asciidoc # metricbeat/metricbeat.reference.yml # metricbeat/module/kubernetes/_meta/config.reference.yml # metricbeat/module/kubernetes/_meta/config.yml # metricbeat/modules.d/kubernetes.yml.disabled # x-pack/metricbeat/metricbeat.reference.yml
elastic · Dec 20, 2022 · 9089e58 · 9089e58
1 parent 8f33651
commit 9089e58
Show file tree

Hide file tree

Showing 11 changed files with 300 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -81,6 +81,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Metricbeat*
 
+- Azure Billing: upgrade Usage Details API to version 2019-10-01 {pull}31970[31970]
+- Azure Billing: switch to Cost Management API for forecast data {pull}32589[32589]
+- Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}33166[33166]
+- Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308]
+- Add per-thread metrics to system_summary {pull}33614[33614]
+- Add GCP CloudSQL metadata {pull}33066[33066]
+- Remove GCP Compute metadata cache {pull}33655[33655]
+- Add support for multiple regions in GCP {pull}32964[32964]
+- Add GCP Redis regions support {pull}33728[33728]
+- Add namespace metadata to all namespaced kubernetes resources. {pull}33763[33763]
 
 *Packetbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -9710,6 +9710,7 @@ SOFTWARE.
 
 
 --------------------------------------------------------------------------------
+<<<<<<< HEAD
 Dependency : github.com/gocarina/gocsv
 Version: v0.0.0-20170324095351-ffef3ffc77be
 Licence type (autodetected): MIT
@@ -10134,6 +10135,14 @@ Licence type (autodetected): Apache-2.0
 
 Contents of probable licence file $GOMODCACHE/github.com/golang/[email protected]/LICENSE:
 
+=======
+Dependency : github.com/elastic/elastic-agent-autodiscover
+Version: v0.5.0
+Licence type (autodetected): Apache-2.0
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/go.mod b/go.mod
@@ -196,8 +196,35 @@ require (
 )
 
 require (
+<<<<<<< HEAD
 	github.com/elastic/elastic-agent-libs v0.2.11
 	github.com/elastic/elastic-agent-system-metrics v0.4.4
+=======
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
+	github.com/Azure/go-autorest/autorest/adal v0.9.14
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17
+	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4
+	github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8
+	github.com/aws/smithy-go v1.12.0
+	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
+	github.com/elastic/bayeux v1.0.5
+	github.com/elastic/elastic-agent-autodiscover v0.5.0
+	github.com/elastic/elastic-agent-libs v0.2.15
+	github.com/elastic/elastic-agent-shipper-client v0.4.0
+	github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51
+	github.com/elastic/go-elasticsearch/v8 v8.2.0
+	github.com/elastic/mito v0.0.0-20221207004749-2f0f2875e464
+	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
+	github.com/google/cel-go v0.12.5
+	github.com/googleapis/gax-go/v2 v2.5.1
+	github.com/pierrec/lz4/v4 v4.1.15
+	github.com/shirou/gopsutil/v3 v3.21.12
+	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
+	go.elastic.co/apm/module/apmhttp/v2 v2.0.0
+	go.elastic.co/apm/v2 v2.0.0
+	go.mongodb.org/mongo-driver v1.5.1
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -323,11 +323,18 @@ github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible h1:jlUO91EFZu
 github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible/go.mod h1:4zuQekLQi489ShcqTmS1Zj1ta0qrcNBlSuGa+ziu2vM=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
+<<<<<<< HEAD
 github.com/elastic/ecs v1.12.0 h1:u6WZ2AWtxv5vHvTQ4EuVZdWZ51mKHQ2UIltRePcta5U=
 github.com/elastic/ecs v1.12.0/go.mod h1:pgiLbQsijLOJvFR8OTILLu0Ni/R/foUNg0L+T6mU9b4=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
 github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4=
+=======
+github.com/elastic/elastic-agent-autodiscover v0.5.0 h1:fiDJQKB148RsRISqqbGINtJ/ZPupeVjwn0fbPz9ZDOc=
+github.com/elastic/elastic-agent-autodiscover v0.5.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4=
+github.com/elastic/elastic-agent-client/v7 v7.0.2-0.20221129150247-15881a8e64ef h1:+3AWaimDL826eoU06qOFBtA3xmyuTr9YUMVWvnim4mU=
+github.com/elastic/elastic-agent-client/v7 v7.0.2-0.20221129150247-15881a8e64ef/go.mod h1:cHviLpA5fAwMbfBIHBVNl16qp90bO7pKHMAQaG+9raU=
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
 github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0=
 github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=

diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc
@@ -231,6 +231,23 @@ metricbeat.modules:
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
+=======
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+  # By default all labels will be included while annotations are not added by default.
+  # add_resource_metadata:
+  #   namespace:
+  #     include_labels: ["namespacelabel1"]
+  #   node:
+  #     include_labels: ["nodelabel2"]
+  #     include_annotations: ["nodeannotation1"]
+  #   deployment: false
+  #   cronjob: false
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
 # State metrics from kube-state-metrics service:
 - module: kubernetes
@@ -261,6 +278,47 @@ metricbeat.modules:
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
+=======
+  # Set the namespace to watch for resources
+  #namespace: staging
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+  # By default all labels will be included while annotations are not added by default.
+  # add_resource_metadata:
+  #   namespace:
+  #     include_labels: ["namespacelabel1"]
+  #   node:
+  #     include_labels: ["nodelabel2"]
+  #     include_annotations: ["nodeannotation1"]
+  #   deployment: false
+  #   cronjob: false
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+
+# Kubernetes Events
+- module: kubernetes
+  enabled: true
+  metricsets:
+    - event
+  period: 10s
+  # Skip events older than Metricbeat's statup time is enabled by default.
+  # Setting to false the skip_older setting will stop filtering older events.
+  # This setting is also useful went Event's timestamps are not populated properly.
+  #skip_older: false
+  # If kube_config is not set, KUBECONFIG environment variable will be checked
+  # and if not present it will fall back to InCluster
+  #kube_config: ~/.kube/config
+  # Set the namespace to watch for events
+  #namespace: staging
+  # Set the sync period of the watchers
+  #sync_period: 10m
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
 # Kubernetes API server
 # (when running metricbeat as a deployment)

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -500,6 +500,23 @@ metricbeat.modules:
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
+=======
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+  # By default all labels will be included while annotations are not added by default.
+  # add_resource_metadata:
+  #   namespace:
+  #     include_labels: ["namespacelabel1"]
+  #   node:
+  #     include_labels: ["nodelabel2"]
+  #     include_annotations: ["nodeannotation1"]
+  #   deployment: false
+  #   cronjob: false
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
 # State metrics from kube-state-metrics service:
 - module: kubernetes
@@ -530,6 +547,47 @@ metricbeat.modules:
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
+=======
+  # Set the namespace to watch for resources
+  #namespace: staging
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+  # By default all labels will be included while annotations are not added by default.
+  # add_resource_metadata:
+  #   namespace:
+  #     include_labels: ["namespacelabel1"]
+  #   node:
+  #     include_labels: ["nodelabel2"]
+  #     include_annotations: ["nodeannotation1"]
+  #   deployment: false
+  #   cronjob: false
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+
+# Kubernetes Events
+- module: kubernetes
+  enabled: true
+  metricsets:
+    - event
+  period: 10s
+  # Skip events older than Metricbeat's statup time is enabled by default.
+  # Setting to false the skip_older setting will stop filtering older events.
+  # This setting is also useful went Event's timestamps are not populated properly.
+  #skip_older: false
+  # If kube_config is not set, KUBECONFIG environment variable will be checked
+  # and if not present it will fall back to InCluster
+  #kube_config: ~/.kube/config
+  # Set the namespace to watch for events
+  #namespace: staging
+  # Set the sync period of the watchers
+  #sync_period: 10m
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
 # Kubernetes API server
 # (when running metricbeat as a deployment)

diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml
@@ -23,6 +23,23 @@
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
+=======
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+  # By default all labels will be included while annotations are not added by default.
+  # add_resource_metadata:
+  #   namespace:
+  #     include_labels: ["namespacelabel1"]
+  #   node:
+  #     include_labels: ["nodelabel2"]
+  #     include_annotations: ["nodeannotation1"]
+  #   deployment: false
+  #   cronjob: false
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
 # State metrics from kube-state-metrics service:
 - module: kubernetes
@@ -53,6 +70,47 @@
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
+=======
+  # Set the namespace to watch for resources
+  #namespace: staging
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+  # By default all labels will be included while annotations are not added by default.
+  # add_resource_metadata:
+  #   namespace:
+  #     include_labels: ["namespacelabel1"]
+  #   node:
+  #     include_labels: ["nodelabel2"]
+  #     include_annotations: ["nodeannotation1"]
+  #   deployment: false
+  #   cronjob: false
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+
+# Kubernetes Events
+- module: kubernetes
+  enabled: true
+  metricsets:
+    - event
+  period: 10s
+  # Skip events older than Metricbeat's statup time is enabled by default.
+  # Setting to false the skip_older setting will stop filtering older events.
+  # This setting is also useful went Event's timestamps are not populated properly.
+  #skip_older: false
+  # If kube_config is not set, KUBECONFIG environment variable will be checked
+  # and if not present it will fall back to InCluster
+  #kube_config: ~/.kube/config
+  # Set the namespace to watch for events
+  #namespace: staging
+  # Set the sync period of the watchers
+  #sync_period: 10m
+  # Kubernetes client QPS and burst can be configured additionally
+  #kube_client_options:
+  #  qps: 5
+  #  burst: 10
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
 
 # Kubernetes API server
 # (when running metricbeat as a deployment)

diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml
@@ -23,8 +23,14 @@
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
   # To configure additionally node and namespace metadata, added to pod, service and container resource types,
   # `add_resource_metadata` can be defined.
+=======
+  # Set the namespace to watch for resources
+  #namespace: staging
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:
   #   namespace:

diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go
@@ -99,10 +99,12 @@ func NewResourceMetadataEnricher(
 
 	cfg, _ := common.NewConfigFrom(&commonMetaConfig)
 
-	metaGen := metadata.NewResourceMetadataGenerator(cfg, watcher.Client())
 	podMetaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, config.AddResourceMetadata)
 	namespaceMeta := metadata.NewNamespaceMetadataGenerator(config.AddResourceMetadata.Namespace, namespaceWatcher.Store(), watcher.Client())
 	serviceMetaGen := metadata.NewServiceMetadataGenerator(cfg, watcher.Store(), namespaceMeta, watcher.Client())
+
+	metaGen := metadata.NewNamespaceAwareResourceMetadataGenerator(cfg, watcher.Client(), namespaceMeta)
+
 	enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher,
 		// update
 		func(m map[string]common.MapStr, r kubernetes.Resource) {

diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled
@@ -26,8 +26,14 @@
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+<<<<<<< HEAD
   # To configure additionally node and namespace metadata, added to pod, service and container resource types,
   # `add_resource_metadata` can be defined.
+=======
+  # Set the namespace to watch for resources
+  #namespace: staging
+  # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
+>>>>>>> 951b244e0e (Use NamespaceAwareResourceMetaGenerator for all generic kubernetes resources (#33763))
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:
   #   namespace: