Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

allows disable pod events enrichment with deployment name #28521

Merged
merged 9 commits into from
Nov 8, 2021
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280]
- Allow `clone3` syscall in seccomp filters. {pull}28117[28117]
- Remove unnecessary escaping step in dashboard loading, so they can be displayed in Kibana. {pull}28395[28395]
- Allows disable pod events enrichment with deployment name {pull}28521[28521]
- Fix AWS proxy_url config from url to string type. {pull}28725[28725]
- Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683]

Expand Down
2 changes: 1 addition & 1 deletion libbeat/autodiscover/providers/kubernetes/pod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2023,7 +2023,7 @@ func TestPod_EmitEvent(t *testing.T) {
t.Fatal(err)
}

metaGen := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, client, nil, nil)
metaGen := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, client, nil, nil, true)
p := &Provider{
config: defaultConfig(),
bus: bus.New(logp.NewLogger("bus"), "test"),
Expand Down
10 changes: 6 additions & 4 deletions libbeat/common/kubernetes/metadata/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ type Config struct {

// AddResourceMetadataConfig allows adding config for enriching additional resources
type AddResourceMetadataConfig struct {
Node *common.Config `config:"node"`
Namespace *common.Config `config:"namespace"`
Node *common.Config `config:"node"`
Namespace *common.Config `config:"namespace"`
Deployment bool `config:"deployment"`
}

// InitDefaults initializes the defaults for the config.
Expand All @@ -52,7 +53,8 @@ func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig {
metaConfig.InitDefaults()
metaCfg, _ := common.NewConfigFrom(&metaConfig)
return &AddResourceMetadataConfig{
Node: metaCfg,
Namespace: metaCfg,
Node: metaCfg,
Namespace: metaCfg,
Deployment: true,
}
}
2 changes: 1 addition & 1 deletion libbeat/common/kubernetes/metadata/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func GetPodMetaGen(
if namespaceWatcher != nil && metaConf.Namespace.Enabled() {
namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client())
}
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen)
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, metaConf.Deployment)

return metaGen
}
Expand Down
37 changes: 21 additions & 16 deletions libbeat/common/kubernetes/metadata/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,12 @@ import (
)

type pod struct {
store cache.Store
client k8s.Interface
node MetaGen
namespace MetaGen
resource *Resource
store cache.Store
client k8s.Interface
node MetaGen
namespace MetaGen
resource *Resource
addDeployment bool
}

// NewPodMetadataGenerator creates a metagen for pod resources
Expand All @@ -42,13 +43,15 @@ func NewPodMetadataGenerator(
pods cache.Store,
client k8s.Interface,
node MetaGen,
namespace MetaGen) MetaGen {
namespace MetaGen,
addDeploymentMeta bool) MetaGen {
return &pod{
resource: NewResourceMetadataGenerator(cfg, client),
store: pods,
node: node,
namespace: namespace,
client: client,
resource: NewResourceMetadataGenerator(cfg, client),
store: pods,
node: node,
namespace: namespace,
client: client,
addDeployment: addDeploymentMeta,
}
}

Expand Down Expand Up @@ -84,11 +87,13 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
out := p.resource.GenerateK8s("pod", obj, opts...)

// check if Pod is handled by a ReplicaSet which is controlled by a Deployment
rsName, _ := out.GetValue("replicaset.name")
if rsName, ok := rsName.(string); ok {
dep := p.getRSDeployment(rsName, po.GetNamespace())
if dep != "" {
out.Put("deployment.name", dep)
if p.addDeployment {
rsName, _ := out.GetValue("replicaset.name")
if rsName, ok := rsName.(string); ok {
dep := p.getRSDeployment(rsName, po.GetNamespace())
if dep != "" {
out.Put("deployment.name", dep)
}
}
}

Expand Down
6 changes: 3 additions & 3 deletions libbeat/common/kubernetes/metadata/pod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ func TestPod_Generate(t *testing.T) {
})
assert.NoError(t, err)

metagen := NewPodMetadataGenerator(config, nil, client, nil, nil)
metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, true)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.output, metagen.Generate(test.input))
Expand Down Expand Up @@ -510,7 +510,7 @@ func TestPod_GenerateFromName(t *testing.T) {
assert.NoError(t, err)
pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
pods.Add(test.input)
metagen := NewPodMetadataGenerator(config, pods, client, nil, nil)
metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, true)

accessor, err := meta.Accessor(test.input)
require.NoError(t, err)
Expand Down Expand Up @@ -634,7 +634,7 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
namespaces.Add(test.namespace)
nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client)

metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta)
metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, true)
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.output, metagen.Generate(test.input))
})
Expand Down
4 changes: 3 additions & 1 deletion libbeat/docs/shared-autodiscover.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,8 @@ endif::[]
`add_resource_metadata` can be done for `node` or `namespace`. By default all labels will be included
while annotations are not added by default. This settings are useful when labels' and annotations'
storing requires special handling to avoid overloading the storage output. The enrichment of `node` or `namespace` metadata
can be individually disabled by setting `enabled: false`.
can be individually disabled by setting `enabled: false`. If resource is `pod` and it is created from a `deployment`, by default
the deployment name is added, this can be disabled by set to `false`.
Example:

["source","yaml",subs="attributes"]
Expand All @@ -267,6 +268,7 @@ endif::[]
node:
include_labels: ["nodelabel2"]
include_annotations: ["nodeannotation1"]
deployment: false
-------------------------------------------------------------------------------------

`unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into
Expand Down
8 changes: 4 additions & 4 deletions libbeat/processors/add_kubernetes_metadata/indexers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
"github.com/elastic/beats/v7/libbeat/common/kubernetes"
)

var metagen = metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil)
var metagen = metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, true)

func TestPodIndexer(t *testing.T) {
var testConfig = common.NewConfig()
Expand Down Expand Up @@ -92,7 +92,7 @@ func TestPodIndexer(t *testing.T) {
func TestPodUIDIndexer(t *testing.T) {
var testConfig = common.NewConfig()

metaGenWithPodUID := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil)
metaGenWithPodUID := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, true)

podUIDIndexer, err := NewPodUIDIndexer(*testConfig, metaGenWithPodUID)
assert.NoError(t, err)
Expand Down Expand Up @@ -307,7 +307,7 @@ func TestFilteredGenMeta(t *testing.T) {
})
assert.NoError(t, err)

filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil)
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, true)

podIndexer, err = NewPodNameIndexer(*testConfig, filteredGen)
assert.NoError(t, err)
Expand Down Expand Up @@ -344,7 +344,7 @@ func TestFilteredGenMetaExclusion(t *testing.T) {
})
assert.NoError(t, err)

filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil)
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, true)

podIndexer, err := NewPodNameIndexer(*testConfig, filteredGen)
assert.NoError(t, err)
Expand Down
4 changes: 2 additions & 2 deletions metricbeat/module/kubernetes/util/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func NewResourceMetadataEnricher(
cfg, _ := common.NewConfigFrom(&metaConfig)

metaGen := metadata.NewResourceMetadataGenerator(cfg, watcher.Client())
podMetaGen := metadata.NewPodMetadataGenerator(cfg, nil, watcher.Client(), nil, nil)
podMetaGen := metadata.NewPodMetadataGenerator(cfg, nil, watcher.Client(), nil, nil, true)
serviceMetaGen := metadata.NewServiceMetadataGenerator(cfg, nil, nil, watcher.Client())
enricher := buildMetadataEnricher(watcher,
// update
Expand Down Expand Up @@ -233,7 +233,7 @@ func NewContainerMetadataEnricher(

cfg, _ := common.NewConfigFrom(&metaConfig)

metaGen := metadata.NewPodMetadataGenerator(cfg, nil, watcher.Client(), nil, nil)
metaGen := metadata.NewPodMetadataGenerator(cfg, nil, watcher.Client(), nil, nil, true)
enricher := buildMetadataEnricher(watcher,
// update
func(m map[string]common.MapStr, r kubernetes.Resource) {
Expand Down