From e1bde3afe1c60868046a3f0d725d71c602f39deb Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 31 Aug 2021 16:49:59 +0300 Subject: [PATCH 1/5] Skip add_kubernetes_metadata processor if kubernetes metadata are already there --- .../add_kubernetes_metadata/kubernetes.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index e3ecf49ff35..66c7f96b43f 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -88,6 +88,14 @@ func isKubernetesAvailableWithRetry(client k8sclient.Interface) bool { } } +// kubernetesMetadataExist checks whether an event is already enriched with kubernetes metadata +func kubernetesMetadataExist(event *beat.Event) bool { + if _, err := event.GetValue("kubernetes.namespace"); err != nil { + return false + } + return true +} + // New constructs a new add_kubernetes_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { config, err := newProcessorConfig(cfg, Indexing) @@ -251,12 +259,15 @@ func (k *kubernetesAnnotator) Run(event *beat.Event) (*beat.Event, error) { if !k.kubernetesAvailable { return event, nil } + if kubernetesMetadataExist(event) { + k.log.Debug("Skipping add_kubernetes_metadata processor as kubernetes metadata already exist") + return event, nil + } index := k.matchers.MetadataIndex(event.Fields) if index == "" { k.log.Debug("No container match string, not adding kubernetes data") return event, nil } - k.log.Debugf("Using the following index key %s", index) metadata := k.cache.get(index) if metadata == nil { From 4f75d6c1db569aac11a3a0f942547e786da9f594 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Wed, 1 Sep 2021 16:21:50 +0300 Subject: [PATCH 2/5] Add removed line --- libbeat/processors/add_kubernetes_metadata/kubernetes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index 66c7f96b43f..ab219761ef5 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -268,6 +268,7 @@ func (k *kubernetesAnnotator) Run(event *beat.Event) (*beat.Event, error) { k.log.Debug("No container match string, not adding kubernetes data") return event, nil } + k.log.Debugf("Using the following index key %s", index) metadata := k.cache.get(index) if metadata == nil { From dfa3c640bfdd0493e55ff63e7222c406f2a04917 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Wed, 1 Sep 2021 16:24:51 +0300 Subject: [PATCH 3/5] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 75b21786701..2cf990595dc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -22,6 +22,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add daemonset.name in pods controlled by DaemonSets {pull}26808[26808], {issue}25816[25816] - Kubernetes autodiscover fails in node scope if node name cannot be discovered {pull}26947[26947] - Loading Kibana assets (dashboards, index templates) rely on Saved Object API. So to provide a reliable service, Beats can only import and export dasbhboards using at least Kibana 7.15. {issue}20672[20672] {pull}27220[27220] +- Skip add_kubernetes_metadata processor when kubernetes metadata are already present {pull}27689[27689] *Auditbeat* From b6d4d538e4e019b41ae360aa8f3129b6fd2aaea5 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Thu, 2 Sep 2021 12:35:45 +0300 Subject: [PATCH 4/5] Add test for skipping of the annotator --- .../processors/add_kubernetes_metadata/kubernetes.go | 2 +- .../add_kubernetes_metadata/kubernetes_test.go | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index ab219761ef5..d9302bebd75 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -90,7 +90,7 @@ func isKubernetesAvailableWithRetry(client k8sclient.Interface) bool { // kubernetesMetadataExist checks whether an event is already enriched with kubernetes metadata func kubernetesMetadataExist(event *beat.Event) bool { - if _, err := event.GetValue("kubernetes.namespace"); err != nil { + if _, err := event.GetValue("kubernetes"); err != nil { return false } return true diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go index ed77c8a5d18..063beedcef1 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go @@ -29,8 +29,8 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -// Test metadata updates don't replace existing pod metrics -func TestAnnotatorDeepUpdate(t *testing.T) { +// Test Annotator is skipped if kubernetes metadata already exist +func TestAnnotatorSkipped(t *testing.T) { cfg := common.MustNewConfigFrom(map[string]interface{}{ "lookup_fields": []string{"kubernetes.pod.name"}, }) @@ -53,8 +53,7 @@ func TestAnnotatorDeepUpdate(t *testing.T) { "kubernetes": common.MapStr{ "pod": common.MapStr{ "labels": common.MapStr{ - "dont": "replace", - "original": "fields", + "added": "should not", }, }, }, @@ -85,10 +84,6 @@ func TestAnnotatorDeepUpdate(t *testing.T) { "a": 1, "b": 2, }, - "labels": common.MapStr{ - "dont": "replace", - "original": "fields", - }, }, }, }, event.Fields) From 3955e717adb8a1204464de756fcba91d7df89d75 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Fri, 3 Sep 2021 09:52:19 +0300 Subject: [PATCH 5/5] Trigger Build