From 405b4d2b4626e9d3daee88be5b7afaf095eb61fa Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 11:33:12 -0700 Subject: [PATCH 01/10] Allow users to just set monitoring.cluster_uuid --- libbeat/cmd/instance/beat.go | 58 ++++++++++++++++++++------------ libbeat/monitoring/monitoring.go | 39 ++++++++++++++++----- 2 files changed, 67 insertions(+), 30 deletions(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index ba799d9ab7d..a7b06edfdc7 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -411,32 +411,11 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { return err } - monitoringCfg, reporterSettings, err := monitoring.SelectConfig(b.Config.MonitoringBeatConfig) + err = b.setupMonitoring(settings) if err != nil { return err } - if monitoringCfg.Enabled() { - settings := report.Settings{ - DefaultUsername: settings.Monitoring.DefaultUsername, - Format: reporterSettings.Format, - ClusterUUID: reporterSettings.ClusterUUID, - } - reporter, err := report.New(b.Info, settings, monitoringCfg, b.Config.Output) - if err != nil { - return err - } - defer reporter.Stop() - - // Expose monitoring.cluster_uuid in state API - if reporterSettings.ClusterUUID != "" { - stateRegistry := monitoring.GetNamespace("state").GetRegistry() - monitoringRegistry := stateRegistry.NewRegistry("monitoring") - clusterUUIDRegVar := monitoring.NewString(monitoringRegistry, "cluster_uuid") - clusterUUIDRegVar.Set(reporterSettings.ClusterUUID) - } - } - if b.Config.MetricLogging == nil || b.Config.MetricLogging.Enabled() { reporter, err := log.MakeReporter(b.Info, b.Config.MetricLogging) if err != nil { @@ -896,6 +875,41 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err return callback, nil } + +func (b *Beat) setupMonitoring(settings Settings) error { + monitoringCfg, reporterSettings, err := monitoring.SelectConfig(b.Config.MonitoringBeatConfig) + if err != nil { + return err + } + + monitoringClusterUUID, err := monitoring.GetMonitoringClusterUUID(monitoringCfg) + if err != nil { + return err + } + + if monitoringClusterUUID != "" { + stateRegistry := monitoring.GetNamespace("state").GetRegistry() + monitoringRegistry := stateRegistry.NewRegistry("monitoring") + clusterUUIDRegVar := monitoring.NewString(monitoringRegistry, "cluster_uuid") + clusterUUIDRegVar.Set(monitoringClusterUUID) + } + + if monitoring.IsEnabled(monitoringCfg) { + settings := report.Settings{ + DefaultUsername: settings.Monitoring.DefaultUsername, + Format: reporterSettings.Format, + ClusterUUID: monitoringClusterUUID, + } + reporter, err := report.New(b.Info, settings, monitoringCfg, b.Config.Output) + if err != nil { + return err + } + defer reporter.Stop() + } + + return nil +} + // handleError handles the given error by logging it and then returning the // error. If the err is nil or is a GracefulExit error then the method will // return nil without logging anything. diff --git a/libbeat/monitoring/monitoring.go b/libbeat/monitoring/monitoring.go index e7f13fb69ea..55b58a89dad 100644 --- a/libbeat/monitoring/monitoring.go +++ b/libbeat/monitoring/monitoring.go @@ -19,7 +19,6 @@ package monitoring import ( "errors" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/monitoring/report" @@ -97,14 +96,38 @@ func SelectConfig(beatCfg BeatConfig) (*common.Config, *report.Settings, error) return monitoringCfg, &report.Settings{Format: report.FormatXPackMonitoringBulk}, nil case beatCfg.Monitoring.Enabled(): monitoringCfg := beatCfg.Monitoring - var info struct { - ClusterUUID string `config:"cluster_uuid"` - } - if err := monitoringCfg.Unpack(&info); err != nil { - return nil, nil, err - } - return monitoringCfg, &report.Settings{Format: report.FormatBulk, ClusterUUID: info.ClusterUUID}, nil + return monitoringCfg, &report.Settings{Format: report.FormatBulk}, nil default: return nil, nil, nil } } + +func GetMonitoringClusterUUID(monitoringCfg *common.Config) (string, error) { + if monitoringCfg == nil { + return "", nil + } + + var config struct { + ClusterUUID string `config:"cluster_uuid"` + } + if err := monitoringCfg.Unpack(&config); err != nil { + return "", err + } + + return config.ClusterUUID, nil +} + +func IsEnabled(monitoringCfg *common.Config) bool { + if monitoringCfg == nil { + return false + } + + // If the only setting in the monitoring config is cluster_uuid, it is + // not enabled + fields := monitoringCfg.GetFields() + if len(fields) == 1 && fields[0] == "cluster_uuid" { + return false + } + + return monitoringCfg.Enabled() +} From 4eb851a0e5796943392bce57f2496d4110415b79 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 11:37:44 -0700 Subject: [PATCH 02/10] Adding back comment lost in refactoring --- libbeat/cmd/instance/beat.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index a7b06edfdc7..0a82f4e2d73 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -887,6 +887,7 @@ func (b *Beat) setupMonitoring(settings Settings) error { return err } + // Expose monitoring.cluster_uuid in state API if monitoringClusterUUID != "" { stateRegistry := monitoring.GetNamespace("state").GetRegistry() monitoringRegistry := stateRegistry.NewRegistry("monitoring") From 5c3383ffa241716f1f6d96f87e815954bd5d629e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 11:44:48 -0700 Subject: [PATCH 03/10] Adds godoc comments for functions --- libbeat/monitoring/monitoring.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libbeat/monitoring/monitoring.go b/libbeat/monitoring/monitoring.go index 55b58a89dad..e585858eebf 100644 --- a/libbeat/monitoring/monitoring.go +++ b/libbeat/monitoring/monitoring.go @@ -102,6 +102,7 @@ func SelectConfig(beatCfg BeatConfig) (*common.Config, *report.Settings, error) } } +// GetMonitoringClusterUUID returns the value of the monitoring.cluster_uuid setting, if it is set. func GetMonitoringClusterUUID(monitoringCfg *common.Config) (string, error) { if monitoringCfg == nil { return "", nil @@ -117,6 +118,7 @@ func GetMonitoringClusterUUID(monitoringCfg *common.Config) (string, error) { return config.ClusterUUID, nil } +// IsEnabled returns whether the monitoring reporter is enabled or not. func IsEnabled(monitoringCfg *common.Config) bool { if monitoringCfg == nil { return false From f399aa3a1146c8c20cc3ffa33da2fc05b31f6aba Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 11:45:36 -0700 Subject: [PATCH 04/10] Removes stutter in function name --- libbeat/cmd/instance/beat.go | 2 +- libbeat/monitoring/monitoring.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 0a82f4e2d73..84f2c2c9f63 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -882,7 +882,7 @@ func (b *Beat) setupMonitoring(settings Settings) error { return err } - monitoringClusterUUID, err := monitoring.GetMonitoringClusterUUID(monitoringCfg) + monitoringClusterUUID, err := monitoring.GetClusterUUID(monitoringCfg) if err != nil { return err } diff --git a/libbeat/monitoring/monitoring.go b/libbeat/monitoring/monitoring.go index e585858eebf..5d176ee3d45 100644 --- a/libbeat/monitoring/monitoring.go +++ b/libbeat/monitoring/monitoring.go @@ -102,8 +102,8 @@ func SelectConfig(beatCfg BeatConfig) (*common.Config, *report.Settings, error) } } -// GetMonitoringClusterUUID returns the value of the monitoring.cluster_uuid setting, if it is set. -func GetMonitoringClusterUUID(monitoringCfg *common.Config) (string, error) { +// GetClusterUUID returns the value of the monitoring.cluster_uuid setting, if it is set. +func GetClusterUUID(monitoringCfg *common.Config) (string, error) { if monitoringCfg == nil { return "", nil } From d50360f9123fc74567c68c997e9cb4621e03157d Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 12:09:45 -0700 Subject: [PATCH 05/10] Adding system test --- libbeat/tests/system/config/mockbeat.yml.j2 | 16 ++++++++-- libbeat/tests/system/test_monitoring.py | 35 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 6023545f660..cd3fe5f7dad 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -116,10 +116,20 @@ xpack.monitoring.elasticsearch.state.period: 3s # to speed up tests {% if monitoring -%} #================================ X-Pack Monitoring (direct) ===================================== -monitoring.elasticsearch.hosts: {{monitoring.elasticsearch.hosts}} -monitoring.elasticsearch.metrics.period: 2s # to speed up tests -monitoring.elasticsearch.state.period: 3s # to speed up tests +monitoring: + {% if monitoring.elasticsearch -%} + elasticsearch.hosts: {{monitoring.elasticsearch.hosts}} + elasticsearch.metrics.period: 2s # to speed up tests + elasticsearch.state.period: 3s # to speed up tests + {% endif -%} + + {% if monitoring.cluster_uuid -%} + cluster_uuid: {{monitoring.cluster_uuid}} + {% endif -%} {% endif -%} # vim: set ft=jinja: +{% if http_enabled -%} +http.enabled: {{http_enabled}} +{% endif -%} diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 833cb5cf02e..f6b61a5f86c 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -4,6 +4,9 @@ import re from nose.plugins.attrib import attr import unittest +import requests +import random +import string INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -150,6 +153,29 @@ def test_compare(self): self.assert_same_structure(indirect_beats_state_doc['beats_state'], direct_beats_state_doc['beats_state']) self.assert_same_structure(indirect_beats_stats_doc['beats_stats'], direct_beats_stats_doc['beats_stats']) + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_cluster_uuid_setting(self): + """ + Test that monitoring.cluster_uuid setting may be set without any other monitoring.* settings + """ + test_cluster_uuid = self.random_string(10) + self.render_config_template( + "mockbeat", + monitoring={ + "cluster_uuid": test_cluster_uuid + }, + http_enabled="true" + ) + + proc = self.start_beat(config="mockbeat.yml") + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + + state = self.get_beat_state() + proc.check_kill_and_wait() + + self.assertEqual(test_cluster_uuid, state["monitoring"]["cluster_uuid"]) + def search_monitoring_doc(self, monitoring_type): results = self.es_monitoring.search( index='.monitoring-beats-*', @@ -241,3 +267,12 @@ def get_elasticsearch_monitoring_url(self): host=os.getenv("ES_MONITORING_HOST", "localhost"), port=os.getenv("ES_MONITORING_PORT", "9210") ) + + def get_beat_state(self): + url = "http://localhost:5066/state" + return requests.get(url).json() + + + def random_string(self, size): + char_pool = string.ascii_letters + string.digits + return ''.join(random.choice(char_pool) for i in range(size)) From 988777ad639293478e0a49d50c1092f120166fcf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 12:15:19 -0700 Subject: [PATCH 06/10] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cc38c362513..569259255aa 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -123,6 +123,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Support usage of custom builders without hints and mappers {pull}13839[13839] - Fix memory leak in kubernetes autodiscover provider and add_kubernetes_metadata processor happening when pods are terminated without sending a delete event. {pull}14259[14259] - Fix kubernetes `metaGenerator.ResourceMetadata` when parent reference controller is nil {issue}14320[14320] {pull}14329[14329] +- Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338] *Auditbeat* From 9cc3b1691dc890117b3dd8742f716adde1f28d98 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 12:43:35 -0700 Subject: [PATCH 07/10] Fixing formatting --- libbeat/cmd/instance/beat.go | 1 - libbeat/monitoring/monitoring.go | 1 + libbeat/tests/system/test_monitoring.py | 3 +-- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 84f2c2c9f63..fa1c4fb420a 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -875,7 +875,6 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err return callback, nil } - func (b *Beat) setupMonitoring(settings Settings) error { monitoringCfg, reporterSettings, err := monitoring.SelectConfig(b.Config.MonitoringBeatConfig) if err != nil { diff --git a/libbeat/monitoring/monitoring.go b/libbeat/monitoring/monitoring.go index 5d176ee3d45..ffd21cb118b 100644 --- a/libbeat/monitoring/monitoring.go +++ b/libbeat/monitoring/monitoring.go @@ -19,6 +19,7 @@ package monitoring import ( "errors" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/monitoring/report" diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index f6b61a5f86c..73767e24c7c 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -163,7 +163,7 @@ def test_cluster_uuid_setting(self): self.render_config_template( "mockbeat", monitoring={ - "cluster_uuid": test_cluster_uuid + "cluster_uuid": test_cluster_uuid }, http_enabled="true" ) @@ -272,7 +272,6 @@ def get_beat_state(self): url = "http://localhost:5066/state" return requests.get(url).json() - def random_string(self, size): char_pool = string.ascii_letters + string.digits return ''.join(random.choice(char_pool) for i in range(size)) From 351d7387c102af952e3dd25bcab9a0aa21037a6f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 12 Nov 2019 10:55:27 -0800 Subject: [PATCH 08/10] Move defer to right scope --- libbeat/cmd/instance/beat.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index fa1c4fb420a..d2bfd65f0dd 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -411,10 +411,13 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { return err } - err = b.setupMonitoring(settings) + r, err := b.setupMonitoring(settings) if err != nil { return err } + if r != nil { + defer r.Stop() + } if b.Config.MetricLogging == nil || b.Config.MetricLogging.Enabled() { reporter, err := log.MakeReporter(b.Info, b.Config.MetricLogging) @@ -875,15 +878,15 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err return callback, nil } -func (b *Beat) setupMonitoring(settings Settings) error { +func (b *Beat) setupMonitoring(settings Settings) (report.Reporter, error) { monitoringCfg, reporterSettings, err := monitoring.SelectConfig(b.Config.MonitoringBeatConfig) if err != nil { - return err + return nil, err } monitoringClusterUUID, err := monitoring.GetClusterUUID(monitoringCfg) if err != nil { - return err + return nil, err } // Expose monitoring.cluster_uuid in state API @@ -902,12 +905,12 @@ func (b *Beat) setupMonitoring(settings Settings) error { } reporter, err := report.New(b.Info, settings, monitoringCfg, b.Config.Output) if err != nil { - return err + return nil, err } - defer reporter.Stop() + return reporter, nil } - return nil + return nil, nil } // handleError handles the given error by logging it and then returning the From f91ca8a64d52933cd50b1bd36a35e0b78e5c51b9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 12:09:45 -0700 Subject: [PATCH 09/10] Adding system test --- libbeat/tests/system/test_monitoring.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 73767e24c7c..25f5df647c0 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -272,6 +272,10 @@ def get_beat_state(self): url = "http://localhost:5066/state" return requests.get(url).json() +<<<<<<< HEAD +======= + +>>>>>>> Adding system test def random_string(self, size): char_pool = string.ascii_letters + string.digits return ''.join(random.choice(char_pool) for i in range(size)) From 81d21cdca48056039d9df0ec665fed53def6a5ff Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 12:43:35 -0700 Subject: [PATCH 10/10] Fixing formatting --- libbeat/tests/system/test_monitoring.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 25f5df647c0..73767e24c7c 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -272,10 +272,6 @@ def get_beat_state(self): url = "http://localhost:5066/state" return requests.get(url).json() -<<<<<<< HEAD -======= - ->>>>>>> Adding system test def random_string(self, size): char_pool = string.ascii_letters + string.digits return ''.join(random.choice(char_pool) for i in range(size))