[cmd setup] Add and deprecate setup cmds for index handling (elastic#…

…12132) Deprecate `setup --template` and `setup --ilm` in favour of newly introduced `setup --index-management` command. Fix bug in template and write alias creation order to ensure creating properly managed indices. implements elastic#12095 Co-authored-by: [email protected]
ph · May 21, 2019 · b412f17 · b412f17
1 parent 11d8046
commit b412f17
Show file tree

Hide file tree

Showing 11 changed files with 689 additions and 173 deletions.
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -38,4 +38,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067]
 - Update urllib3 version to 1.24.2 {pull}11930[11930]
 - Add libbeat/common/cleanup package. {pull}12134[12134]
-- Only Load minimal template if no fields are provided. {pull}12103[12103]
+- Only Load minimal template if no fields are provided. {pull}12103[12103]
+- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]
diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
@@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error {
 
 //SetupSettings holds settings necessary for beat setup
 type SetupSettings struct {
-	Template        bool
 	Dashboard       bool
 	MachineLearning bool
 	Pipeline        bool
-	ILMPolicy       bool
+	IndexManagement bool
+	//Deprecated: use IndexManagementKey instead
+	Template bool
+	//Deprecated: use IndexManagementKey instead
+	ILMPolicy bool
 }
 
 // Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
@@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
 			return err
 		}
 
-		if setup.Template || setup.ILMPolicy {
+		if setup.IndexManagement || setup.Template || setup.ILMPolicy {
 			outCfg := b.Config.Output
-
 			if outCfg.Name() != "elasticsearch" {
 				return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled")
 			}
+			esClient, err := elasticsearch.NewConnectedClient(outCfg.Config())
+			if err != nil {
+				return err
+			}
 
-			esConfig := outCfg.Config()
-			if b.IdxSupporter.Enabled() {
-				esClient, err := elasticsearch.NewConnectedClient(esConfig)
-				if err != nil {
-					return err
-				}
-
-				// prepare index by loading templates, lifecycle policies and write aliases
-
-				m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
-				var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
-				if setup.Template {
-					tmplLoadMode = idxmgmt.LoadModeOverwrite
-				}
-				if setup.ILMPolicy {
-					ilmLoadMode = idxmgmt.LoadModeOverwrite
-				}
-
-				err = m.Setup(tmplLoadMode, ilmLoadMode)
-				if err != nil {
-					return err
-				}
+			var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
+			if setup.IndexManagement || setup.Template {
+				loadTemplate = idxmgmt.LoadModeOverwrite
+			}
+			if setup.IndexManagement || setup.ILMPolicy {
+				loadILM = idxmgmt.LoadModeOverwrite
+			}
+			m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
+			if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok {
+				fmt.Println(warn)
+			}
+			if err = m.Setup(loadTemplate, loadILM); err != nil {
+				return err
 			}
-			fmt.Println("Index setup complete.")
+			fmt.Println("Index setup finished.")
 		}
 
 		if setup.Dashboard {

diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go
@@ -28,15 +28,23 @@ import (
 )
 
 const (
-	//TemplateKey used for defining template in setup cmd
-	TemplateKey = "template"
 	//DashboardKey used for registering dashboards in setup cmd
 	DashboardKey = "dashboards"
 	//MachineLearningKey used for registering ml jobs in setup cmd
 	MachineLearningKey = "machine-learning"
 	//PipelineKey used for registering pipelines in setup cmd
 	PipelineKey = "pipelines"
-	//ILMPolicyKey used for registering ilm in setup cmd
+	//IndexManagementKey used for loading all components related to ES index management in setup cmd
+	IndexManagementKey = "index-management"
+
+	//TemplateKey used for loading template in setup cmd
+	//
+	//Deprecated: use IndexManagementKey instead
+	TemplateKey = "template"
+
+	//ILMPolicyKey used for loading ilm in setup cmd
+	//
+	//Deprecated: use IndexManagementKey instead
 	ILMPolicyKey = "ilm-policy"
 )
 
@@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 			}
 
 			var registeredFlags = map[string]bool{
-				TemplateKey:        false,
 				DashboardKey:       false,
 				MachineLearningKey: false,
 				PipelineKey:        false,
+				IndexManagementKey: false,
+				TemplateKey:        false,
 				ILMPolicyKey:       false,
 			}
 			var setupAll = true
@@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 			for k, v := range registeredFlags {
 				if setupAll || v {
 					switch k {
-					case TemplateKey:
-						s.Template = true
 					case DashboardKey:
 						s.Dashboard = true
 					case MachineLearningKey:
 						s.MachineLearning = true
 					case PipelineKey:
 						s.Pipeline = true
+					case IndexManagementKey:
+						s.IndexManagement = true
 					case ILMPolicyKey:
 						s.ILMPolicy = true
+					case TemplateKey:
+						s.Template = true
 					}
 				}
 			}
@@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 		},
 	}
 
-	setup.Flags().Bool(TemplateKey, false, "Setup index template")
 	setup.Flags().Bool(DashboardKey, false, "Setup dashboards")
 	setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations")
 	setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines")
+	setup.Flags().Bool(IndexManagementKey, false,
+		"Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias")
+	setup.Flags().Bool(TemplateKey, false, "Setup index template")
+	setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
 	setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy")
+	setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
 
 	return &setup
 }
diff --git a/libbeat/idxmgmt/componenttype_string.go b/libbeat/idxmgmt/componenttype_string.go
diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go
@@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er
 // A manager instantiated via Supporter is responsible for instantiating/configuring
 // the index throughout the Elastic Stack.
 type Supporter interface {
-	// Enalbed checks if index management is configured to configure templates,
-	// ILM, or aliases.
+	// Enabled checks if index management is configured to setup templates or ILM
 	Enabled() bool
 
 	// BuildSelector create an index selector.
@@ -62,6 +61,9 @@ type Asseter interface {
 // Manager is used to initialize indices, ILM policies, and aliases within the
 // Elastic Stack.
 type Manager interface {
+	VerifySetup(template, ilm LoadMode) (bool, string)
+	// When supporting index lifecycle management, ensure templates and policies
+	// are created before write aliases, to ensure templates are applied to the indices.
 	Setup(template, ilm LoadMode) error
 }