From 4e1e421ed899bef3a5eddaa47897902c2f6e873c Mon Sep 17 00:00:00 2001 From: Nick Mohoric Date: Tue, 27 Nov 2018 15:12:40 -0500 Subject: [PATCH 1/2] Correctly remove the dynamic replicaset from the pod_name --- .../plugin/filter_kubernetes_sumologic.rb | 34 ++- .../test_filter_kubernetes_sumologic.rb | 256 +++++++++++++++++- 2 files changed, 272 insertions(+), 18 deletions(-) diff --git a/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/lib/fluent/plugin/filter_kubernetes_sumologic.rb index e82c5b2..64cfc2a 100644 --- a/lib/fluent/plugin/filter_kubernetes_sumologic.rb +++ b/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -28,6 +28,31 @@ def is_number?(string) true if Float(string) rescue false end + def strip_dynamic_bits(k8s_metadata) + # Strip out dynamic bits from pod name. + # NOTE: Kubernetes deployments append a template hash. + # At the moment this can be in 3 different forms: + # 1) pre-1.8: numeric in pod_template_hash and pod_parts[-2] + # 2) 1.8-1.11: numeric in pod_template_hash, hash in pod_parts[-2] + # 3) post-1.11: hash in pod_template_hash and pod_parts[-2] + + pod_parts = k8s_metadata[:pod].split("-") + pod_template_hash = k8s_metadata[:"label:pod-template-hash"] + if (pod_template_hash == pod_parts[-2] || + to_hash(pod_template_hash) == pod_parts[-2]) + k8s_metadata[:pod_name] = pod_parts[0..-3].join("-") + else + k8s_metadata[:pod_name] = pod_parts[0..-2].join("-") + end + end + + def to_hash(pod_template_hash) + # Convert the pod_template_hash to an alphanumeric string using the same + # logic Kubernetes uses at https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L119 + alphanums = "bcdfghjklmnpqrstvwxz2456789" + pod_template_hash.each_byte.map { |i| alphanums[i.to_i % alphanums.length] }.join("") + end + def filter(tag, time, record) # Set the sumo metadata fields sumo_metadata = record["_sumo_metadata"] || {} @@ -119,14 +144,7 @@ def filter(tag, time, record) end end - # Strip out dynamic bits from pod name. - # NOTE: Kubernetes deployments append a template hash. - pod_parts = k8s_metadata[:pod].split("-") - if is_number?(pod_parts[-2]) - k8s_metadata[:pod_name] = pod_parts[0..-3].join("-") - else - k8s_metadata[:pod_name] = pod_parts[0..-2].join("-") - end + strip_dynamic_bits(k8s_metadata) if annotations["sumologic.com/exclude"] == "true" return nil diff --git a/test/plugin/test_filter_kubernetes_sumologic.rb b/test/plugin/test_filter_kubernetes_sumologic.rb index 03a9d69..e38cc33 100644 --- a/test/plugin/test_filter_kubernetes_sumologic.rb +++ b/test/plugin/test_filter_kubernetes_sumologic.rb @@ -73,7 +73,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -183,7 +183,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -244,7 +244,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log-format-labs-54575ccdb9", + :category => "kubernetes/default/log-format-labs", :host => "", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -289,7 +289,7 @@ def create_driver(conf = CONFIG) "log" => "some message", "stream" => "stdout", "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -350,7 +350,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "", :log_format => "json_merge", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -411,7 +411,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "", :log_format => "text", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -591,7 +591,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "foo", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -656,7 +656,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9", + :category => "kubernetes/default/log/format/labs", :host => "", :log_format => "json", :source => "foo", @@ -782,7 +782,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9/log/format/labs", + :category => "kubernetes/default/log/format/labs/log/format/labs", :host => "", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -843,7 +843,7 @@ def create_driver(conf = CONFIG) "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", }, "_sumo_metadata" => { - :category => "kubernetes/default/log/format/labs/54575ccdb9/undefined", + :category => "kubernetes/default/log/format/labs/undefined", :host => "", :log_format => "json", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", @@ -900,4 +900,240 @@ def create_driver(conf = CONFIG) end assert_equal(0, d.filtered_records.size) end + + test "test_pre_1.8_dynamic_bit_removal" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-1013177865-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-1013177865-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-1013177865-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_1.8-1.11_dynamic_bit_removal" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_post_1.11_dynamic_bit_removal" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_mismatch_dynamic_bit_is_left" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-53575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-53575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs/53575ccdb9", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-53575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end end From f08c7838f8f38fdf654df04f0c7892bba8f54175 Mon Sep 17 00:00:00 2001 From: Nick Mohoric Date: Tue, 4 Dec 2018 10:32:53 -0500 Subject: [PATCH 2/2] Rename strip_dynamic_bits function to sanitize_pod_name. Added permanent github link. --- lib/fluent/plugin/filter_kubernetes_sumologic.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/lib/fluent/plugin/filter_kubernetes_sumologic.rb index 64cfc2a..dbe5daf 100644 --- a/lib/fluent/plugin/filter_kubernetes_sumologic.rb +++ b/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -28,7 +28,7 @@ def is_number?(string) true if Float(string) rescue false end - def strip_dynamic_bits(k8s_metadata) + def sanitize_pod_name(k8s_metadata) # Strip out dynamic bits from pod name. # NOTE: Kubernetes deployments append a template hash. # At the moment this can be in 3 different forms: @@ -47,8 +47,8 @@ def strip_dynamic_bits(k8s_metadata) end def to_hash(pod_template_hash) - # Convert the pod_template_hash to an alphanumeric string using the same - # logic Kubernetes uses at https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L119 + # Convert the pod_template_hash to an alphanumeric string using the same logic Kubernetes + # uses at https://github.com/kubernetes/apimachinery/blob/18a5ff3097b4b189511742e39151a153ee16988b/pkg/util/rand/rand.go#L119 alphanums = "bcdfghjklmnpqrstvwxz2456789" pod_template_hash.each_byte.map { |i| alphanums[i.to_i % alphanums.length] }.join("") end @@ -144,7 +144,7 @@ def filter(tag, time, record) end end - strip_dynamic_bits(k8s_metadata) + sanitize_pod_name(k8s_metadata) if annotations["sumologic.com/exclude"] == "true" return nil