[cmd setup] Add and deprecate setup cmds for index handling

CHANGELOG-developer.next.asciidoc

@@ -37,3 +37,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Jinja2 version to 2.10.1. {pull}11817[11817]
 - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067]
 - Update urllib3 version to 1.24.2 {pull}11930[11930]
+- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]

libbeat/cmd/instance/beat.go

@@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error {
 
 //SetupSettings holds settings necessary for beat setup
 type SetupSettings struct {
-	Template        bool
 	Dashboard       bool
 	MachineLearning bool
 	Pipeline        bool
-	ILMPolicy       bool
+	IndexManagement bool
+	//Deprecated: use IndexManagementKey instead
+	Template bool
+	//Deprecated: use IndexManagementKey instead
+	ILMPolicy bool
 }
 
 // Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
@@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
 			return err
 		}
 
-		if setup.Template || setup.ILMPolicy {
+		if setup.IndexManagement || setup.Template || setup.ILMPolicy {
 			outCfg := b.Config.Output
-
 			if outCfg.Name() != "elasticsearch" {
 				return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled")
 			}
+			esClient, err := elasticsearch.NewConnectedClient(outCfg.Config())
+			if err != nil {
+				return err
+			}
 
-			esConfig := outCfg.Config()
-			if b.IdxSupporter.Enabled() {
-				esClient, err := elasticsearch.NewConnectedClient(esConfig)
-				if err != nil {
-					return err
-				}
-
-				// prepare index by loading templates, lifecycle policies and write aliases
-
-				m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
-				var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
-				if setup.Template {
-					tmplLoadMode = idxmgmt.LoadModeOverwrite
-				}
-				if setup.ILMPolicy {
-					ilmLoadMode = idxmgmt.LoadModeOverwrite
-				}
-
-				err = m.Setup(tmplLoadMode, ilmLoadMode)
-				if err != nil {
-					return err
-				}
+			var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
+			if setup.IndexManagement || setup.Template {
+				loadTemplate = idxmgmt.LoadModeOverwrite
+			}
+			if setup.IndexManagement || setup.ILMPolicy {
+				loadILM = idxmgmt.LoadModeOverwrite
+			}
+			m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
+			if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok {
+				fmt.Println(warn)
+			}
+			if err = m.Setup(loadTemplate, loadILM); err != nil {
+				return err
 			}
-			fmt.Println("Index setup complete.")
+			fmt.Println("Index setup finished.")
 		}
 
 		if setup.Dashboard {

libbeat/cmd/setup.go

@@ -28,15 +28,23 @@ import (
 )
 
 const (
-	//TemplateKey used for defining template in setup cmd
-	TemplateKey = "template"
 	//DashboardKey used for registering dashboards in setup cmd
 	DashboardKey = "dashboards"
 	//MachineLearningKey used for registering ml jobs in setup cmd
 	MachineLearningKey = "machine-learning"
 	//PipelineKey used for registering pipelines in setup cmd
 	PipelineKey = "pipelines"
-	//ILMPolicyKey used for registering ilm in setup cmd
+	//IndexManagementKey used for loading all components related to ES index management in setup cmd
+	IndexManagementKey = "index-management"
+
+	//TemplateKey used for loading template in setup cmd
+	//
+	//Deprecated: use IndexManagementKey instead
+	TemplateKey = "template"
+
+	//ILMPolicyKey used for loading ilm in setup cmd
+	//
+	//Deprecated: use IndexManagementKey instead
 	ILMPolicyKey = "ilm-policy"
 )
 
@@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 			}
 
 			var registeredFlags = map[string]bool{
-				TemplateKey:        false,
 				DashboardKey:       false,
 				MachineLearningKey: false,
 				PipelineKey:        false,
+				IndexManagementKey: false,
+				TemplateKey:        false,
 				ILMPolicyKey:       false,
 			}
 			var setupAll = true
@@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 			for k, v := range registeredFlags {
 				if setupAll || v {
 					switch k {
-					case TemplateKey:
-						s.Template = true
 					case DashboardKey:
 						s.Dashboard = true
 					case MachineLearningKey:
 						s.MachineLearning = true
 					case PipelineKey:
 						s.Pipeline = true
+					case IndexManagementKey:
+						s.IndexManagement = true
 					case ILMPolicyKey:
 						s.ILMPolicy = true
+					case TemplateKey:
+						s.Template = true
 					}
 				}
 			}
@@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 		},
 	}
 
-	setup.Flags().Bool(TemplateKey, false, "Setup index template")
 	setup.Flags().Bool(DashboardKey, false, "Setup dashboards")
 	setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations")
 	setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines")
+	setup.Flags().Bool(IndexManagementKey, false,
+		"Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias")
+	setup.Flags().Bool(TemplateKey, false, "Setup index template")
+	setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
 	setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy")
+	setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
 
 	return &setup
 }

libbeat/idxmgmt/idxmgmt.go

@@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er
 // A manager instantiated via Supporter is responsible for instantiating/configuring
 // the index throughout the Elastic Stack.
 type Supporter interface {
-	// Enalbed checks if index management is configured to configure templates,
-	// ILM, or aliases.
+	// Enabled checks if index management is configured to setup templates or ILM
 	Enabled() bool
 
 	// BuildSelector create an index selector.
@@ -62,6 +61,7 @@ type Asseter interface {
 // Manager is used to initialize indices, ILM policies, and aliases within the
 // Elastic Stack.
 type Manager interface {
+	VerifySetup(template, ilm LoadMode) (bool, string)
 	Setup(template, ilm LoadMode) error
 }
 

libbeat/idxmgmt/std.go

@@ -62,6 +62,33 @@ type ilmIndexSelector struct {
 	st    *indexState
 }
 
+type componentName uint8
+
+//go:generate stringer -linecomment -type componentName
+const (
+	componentTemplate componentName = iota //template
+	componentILM                           //ilm
+)
+
+type component struct {
+	name                     componentName
+	enabled, overwrite, load bool
+}
+
+func newComponent(name componentName, enabled, overwrite bool, mode LoadMode) component {
+	if mode == LoadModeUnset && !enabled {
+		mode = LoadModeDisabled
+	}
+	if mode >= LoadModeOverwrite {
+		overwrite = true
+	}
+	if mode == LoadModeForce {
+		enabled = true
+	}
+	load := mode.Enabled() && enabled
+	return component{name: name, enabled: enabled, overwrite: overwrite, load: load}
+}
+
 func newIndexSupport(
 	log *logp.Logger,
 	info beat.Info,
@@ -95,7 +122,17 @@ func newIndexSupport(
 }
 
 func (s *indexSupport) Enabled() bool {
-	return s.templateCfg.Enabled || (s.ilm.Mode() != ilm.ModeDisabled)
+	return s.enabled(componentTemplate) || s.enabled(componentILM)
+}
+
+func (s *indexSupport) enabled(c componentName) bool {
+	switch c {
+	case componentTemplate:
+		return s.templateCfg.Enabled
+	case componentILM:
+		return s.ilm.Mode() != ilm.ModeDisabled
+	}
+	return false
 }
 
 func (s *indexSupport) Manager(
@@ -176,41 +213,55 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector,
 		st:    &s.st,
 	}, nil
 }
+func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) {
+	ilmComponent := newComponent(componentILM, m.support.enabled(componentILM), false, loadILM)
+	templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate),
+		m.support.templateCfg.Overwrite, loadTemplate)
+
+	if ilmComponent.load && !templateComponent.load {
+		return false, "Loading ILM policy and write alias without loading template " +
+			"is not recommended. Check your configuration."
+	}
+	if templateComponent.load && !ilmComponent.load && ilmComponent.enabled {
+		return false, "Loading template with ILM settings whithout loading ILM policy and alias can lead " +
+			"to issues and is not recommended. Check your configuration"
+	}
+	var warn string
+	if !ilmComponent.load {
+		warn += "ILM policy and write alias loading not enabled. "
+	}
+	if !templateComponent.load {
+		warn += "Template loading not enabled."
+	}
+	return warn == "", warn
+}
 
 func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
-	var err error
 	log := m.support.log
 
-	withILM := m.support.st.withILM.Load()
-	if !withILM {
-		withILM, err = m.ilm.Enabled()
-		if err != nil {
-			return err
-		}
+	withILM, err := m.setupWithILM()
+	if err != nil {
+		return err
 	}
-	if loadILM == LoadModeUnset {
-		if withILM {
-			loadILM = LoadModeEnabled
-			log.Info("Auto ILM enable success.")
-		} else {
-			loadILM = LoadModeDisabled
-		}
+	if withILM && loadILM.Enabled() {
+		log.Info("Auto ILM enable success.")
 	}
 
-	if loadILM == LoadModeForce || withILM && loadILM.Enabled() {
-		// mark ILM as enabled in indexState if withILM is true
-		m.support.st.withILM.CAS(false, true)
+	ilmComponent := newComponent(componentILM, withILM, false, loadILM)
+	templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate),
+		m.support.templateCfg.Overwrite, loadTemplate)
 
+	if ilmComponent.load {
 		// install ilm policy
-		policyCreated, err := m.ilm.EnsurePolicy(loadILM >= LoadModeOverwrite)
+		policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite)
 		if err != nil {
 			return err
 		}
 		log.Info("ILM policy successfully loaded.")
 
 		// The template should be updated if a new policy is created.
-		if policyCreated && loadTemplate.Enabled() {
-			loadTemplate = LoadModeOverwrite
+		if policyCreated && templateComponent.enabled {
+			templateComponent.overwrite = true
 		}
 
 		// create alias
@@ -224,24 +275,17 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 		}
 	}
 
-	// create and install template
-	if loadTemplate == LoadModeForce || m.support.templateCfg.Enabled && loadTemplate.Enabled() {
+	if templateComponent.load {
 		tmplCfg := m.support.templateCfg
+		tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled
 
-		if withILM {
+		if ilmComponent.enabled {
 			ilm := m.support.ilm
 			tmplCfg, err = applyILMSettings(log, tmplCfg, ilm.Policy(), ilm.Alias())
 			if err != nil {
 				return err
 			}
 		}
-
-		if loadTemplate == LoadModeForce {
-			tmplCfg.Enabled = true
-		}
-		if loadTemplate >= LoadModeOverwrite {
-			tmplCfg.Overwrite = true
-		}
 		fields := m.assets.Fields(m.support.info.Beat)
 		err = m.clientHandler.Load(tmplCfg, m.support.info, fields, m.support.migration)
 		if err != nil {
@@ -254,6 +298,22 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 	return nil
 }
 
+func (m *indexManager) setupWithILM() (bool, error) {
+	var err error
+	withILM := m.support.st.withILM.Load()
+	if !withILM {
+		withILM, err = m.ilm.Enabled()
+		if err != nil {
+			return false, err
+		}
+		if withILM {
+			// mark ILM as enabled in indexState
+			m.support.st.withILM.CAS(false, true)
+		}
+	}
+	return withILM, nil
+}
+
 func (s *ilmIndexSelector) Select(evt *beat.Event) (string, error) {
 	if idx := getEventCustomIndex(evt); idx != "" {
 		return idx, nil