Skip to content
This repository has been archived by the owner on Feb 5, 2021. It is now read-only.

Correctly remove the dynamic replicaset from the pod_name #100

Merged
merged 2 commits into from
Dec 6, 2018
Merged

Correctly remove the dynamic replicaset from the pod_name #100

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4e1e421
Correctly remove the dynamic replicaset from the pod_name
nmohoric Nov 27, 2018
f08c783
Rename strip_dynamic_bits function to sanitize_pod_name. Added perman…
nmohoric Dec 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions lib/fluent/plugin/filter_kubernetes_sumologic.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,31 @@ def is_number?(string)
true if Float(string) rescue false
end

def strip_dynamic_bits(k8s_metadata)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd recommend def sanitize_pod_name

# Strip out dynamic bits from pod name.
# NOTE: Kubernetes deployments append a template hash.
# At the moment this can be in 3 different forms:
# 1) pre-1.8: numeric in pod_template_hash and pod_parts[-2]
# 2) 1.8-1.11: numeric in pod_template_hash, hash in pod_parts[-2]
# 3) post-1.11: hash in pod_template_hash and pod_parts[-2]

pod_parts = k8s_metadata[:pod].split("-")
pod_template_hash = k8s_metadata[:"label:pod-template-hash"]
if (pod_template_hash == pod_parts[-2] ||
to_hash(pod_template_hash) == pod_parts[-2])
k8s_metadata[:pod_name] = pod_parts[0..-3].join("-")
else
k8s_metadata[:pod_name] = pod_parts[0..-2].join("-")
end
end

def to_hash(pod_template_hash)
# Convert the pod_template_hash to an alphanumeric string using the same
# logic Kubernetes uses at https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L119
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should use a permanent link here in case this module moves:

https://github.com/kubernetes/apimachinery/blob/18a5ff3097b4b189511742e39151a153ee16988b/pkg/util/rand/rand.go#L119

alphanums = "bcdfghjklmnpqrstvwxz2456789"
pod_template_hash.each_byte.map { |i| alphanums[i.to_i % alphanums.length] }.join("")
end

def filter(tag, time, record)
# Set the sumo metadata fields
sumo_metadata = record["_sumo_metadata"] || {}
Expand Down Expand Up @@ -119,14 +144,7 @@ def filter(tag, time, record)
end
end

# Strip out dynamic bits from pod name.
# NOTE: Kubernetes deployments append a template hash.
pod_parts = k8s_metadata[:pod].split("-")
if is_number?(pod_parts[-2])
k8s_metadata[:pod_name] = pod_parts[0..-3].join("-")
else
k8s_metadata[:pod_name] = pod_parts[0..-2].join("-")
end
strip_dynamic_bits(k8s_metadata)

if annotations["sumologic.com/exclude"] == "true"
return nil
Expand Down
256 changes: 246 additions & 10 deletions test/plugin/test_filter_kubernetes_sumologic.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -183,7 +183,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -244,7 +244,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log-format-labs-54575ccdb9",
:category => "kubernetes/default/log-format-labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -289,7 +289,7 @@ def create_driver(conf = CONFIG)
"log" => "some message",
"stream" => "stdout",
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -350,7 +350,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json_merge",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -411,7 +411,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "text",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -591,7 +591,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "foo",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -656,7 +656,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9",
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "foo",
Expand Down Expand Up @@ -782,7 +782,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9/log/format/labs",
:category => "kubernetes/default/log/format/labs/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -843,7 +843,7 @@ def create_driver(conf = CONFIG)
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/54575ccdb9/undefined",
:category => "kubernetes/default/log/format/labs/undefined",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
Expand Down Expand Up @@ -900,4 +900,240 @@ def create_driver(conf = CONFIG)
end
assert_equal(0, d.filtered_records.size)
end

test "test_pre_1.8_dynamic_bit_removal" do
conf = %{}
d = create_driver(conf)
time = @time
input = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-1013177865-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "1013177865",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
}
d.run do
d.feed("filter.test", time, input)
end
expected = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-1013177865-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "1013177865",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-1013177865-9d677.log-format-labs",
},
}
assert_equal(1, d.filtered_records.size)
assert_equal(d.filtered_records[0], expected)
end

test "test_1.8-1.11_dynamic_bit_removal" do
conf = %{}
d = create_driver(conf)
time = @time
input = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-54575ccdb9-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "1013177865",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
}
d.run do
d.feed("filter.test", time, input)
end
expected = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-54575ccdb9-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "1013177865",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
},
}
assert_equal(1, d.filtered_records.size)
assert_equal(d.filtered_records[0], expected)
end

test "test_post_1.11_dynamic_bit_removal" do
conf = %{}
d = create_driver(conf)
time = @time
input = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-54575ccdb9-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "54575ccdb9",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
}
d.run do
d.feed("filter.test", time, input)
end
expected = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-54575ccdb9-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "54575ccdb9",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs",
},
}
assert_equal(1, d.filtered_records.size)
assert_equal(d.filtered_records[0], expected)
end

test "test_mismatch_dynamic_bit_is_left" do
conf = %{}
d = create_driver(conf)
time = @time
input = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-53575ccdb9-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "54575ccdb9",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
}
d.run do
d.feed("filter.test", time, input)
end
expected = {
"timestamp" => 1538677347823,
"log" => "some message",
"stream" => "stdout",
"docker" => {
"container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0",
},
"kubernetes" => {
"container_name" => "log-format-labs",
"namespace_name" => "default",
"pod_name" => "log-format-labs-53575ccdb9-9d677",
"pod_id" => "170af806-c801-11e8-9009-025000000001",
"labels" => {
"pod-template-hash" => "54575ccdb9",
"run" => "log-format-labs",
},
"host" => "docker-for-desktop",
"master_url" => "https =>//10.96.0.1 =>443/api",
"namespace_id" => "e8572415-9596-11e8-b28b-025000000001",
},
"_sumo_metadata" => {
:category => "kubernetes/default/log/format/labs/53575ccdb9",
:host => "",
:log_format => "json",
:source => "default.log-format-labs-53575ccdb9-9d677.log-format-labs",
},
}
assert_equal(1, d.filtered_records.size)
assert_equal(d.filtered_records[0], expected)
end
end