Fix order of template and write alias creation.

According to elastic#12149
simitt · May 10, 2019 · b5948f2 · b5948f2
1 parent 9810cc5
commit b5948f2
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 43 deletions.
diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go
@@ -62,6 +62,8 @@ type Asseter interface {
 // Elastic Stack.
 type Manager interface {
 	VerifySetup(template, ilm LoadMode) (bool, string)
+	// When supporting index lifecycle management, ensure templates and policies
+	// are created before write aliases, to ensure templates are applied to the indices.
 	Setup(template, ilm LoadMode) error
 }
 

diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go
@@ -240,6 +240,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string
 	return warn == "", warn
 }
 
+//
 func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 	log := m.support.log
 
@@ -267,16 +268,6 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 		if policyCreated && templateComponent.enabled {
 			templateComponent.overwrite = true
 		}
-
-		// create alias
-		if err := m.ilm.EnsureAlias(); err != nil {
-			if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists {
-				return err
-			}
-			log.Info("Write alias exists already")
-		} else {
-			log.Info("Write alias successfully generated.")
-		}
 	}
 
 	if templateComponent.load {
@@ -298,6 +289,18 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 		log.Info("Loaded index template.")
 	}
 
+	if ilmComponent.load {
+		// ensure alias is created after the template is created
+		if err := m.ilm.EnsureAlias(); err != nil {
+			if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists {
+				return err
+			}
+			log.Info("Write alias exists already")
+		} else {
+			log.Info("Write alias successfully generated.")
+		}
+	}
+
 	return nil
 }
 

diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py
@@ -7,17 +7,20 @@ class IdxMgmt(object):
 
     def __init__(self, client, index):
         self._client = client
-        if index == "":
-            index == "mockbeat"
-        self._index = index
+        self._index = index if index != '' and index != '*' else 'mockbeat'
 
     def needs_init(self, s):
         return s == '' or s == '*'
 
-    def delete(self, index=""):
-        self.delete_index_and_alias(index=index)
-        self.delete_template(template=index)
-        self.delete_policy(policy=index)
+    def delete(self, indices=[]):
+        indices = list(filter(lambda x: x != '', indices))
+        if not indices:
+            indices == [self._index]
+        for i in indices:
+            self.delete_index_and_alias(i)
+            self.delete_template(template=i)
+        for i in indices:
+            self.delete_policy(policy=i)
 
     def delete_index_and_alias(self, index=""):
         if self.needs_init(index):

diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py
@@ -3,9 +3,9 @@
 import os
 from nose.plugins.attrib import attr
 import unittest
-import shutil
-import datetime
 import logging
+from nose.tools import raises
+from elasticsearch import RequestError
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
@@ -28,17 +28,13 @@ def setUp(self):
 
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(index=self.custom_policy)
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name])
 
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
     def tearDown(self):
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.custom_policy)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -64,6 +60,26 @@ def test_setup_default(self):
         self.idxmgmt.assert_alias_created(self.alias_name)
         self.idxmgmt.assert_policy_created(self.policy_name)
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    @raises(RequestError)
+    def test_setup_default(self):
+        """
+        Test setup --index-management with default config
+        """
+        self.render_config()
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.cmd])
+
+        assert exit_code == 0
+        self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
+        self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"])
+        self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
+        self.idxmgmt.assert_alias_created(self.alias_name)
+        self.idxmgmt.assert_policy_created(self.policy_name)
+        # try deleting policy needs to raise an error as it is in use
+        self.idxmgmt.delete_policy(self.policy_name)
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_setup_template_disabled(self):

diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py
@@ -21,14 +21,10 @@ def setUp(self):
         self.custom_policy = self.beat_name + "_bar"
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.custom_policy)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name])
 
     def tearDown(self):
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.custom_policy)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -169,17 +165,13 @@ def setUp(self):
         self.custom_policy = self.beat_name + "_bar"
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.custom_policy)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name])
 
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
     def tearDown(self):
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.custom_policy)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name])
 
     def render_config(self, **kwargs):
         self.render_config_template(

diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py
@@ -128,10 +128,10 @@ def setUp(self):
 
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.index_name])
 
     def tearDown(self):
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.index_name])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -186,14 +186,12 @@ def setUp(self):
 
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name])
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
     def tearDown(self):
-        self.idxmgmt.delete(index=self.custom_alias)
-        self.idxmgmt.delete(index=self.index_name)
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name])
 
     def render_config(self, **kwargs):
         self.render_config_template(