diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go index a18eae5d8ff..6837af03d13 100644 --- a/libbeat/idxmgmt/idxmgmt.go +++ b/libbeat/idxmgmt/idxmgmt.go @@ -62,6 +62,8 @@ type Asseter interface { // Elastic Stack. type Manager interface { VerifySetup(template, ilm LoadMode) (bool, string) + // When supporting index lifecycle management, ensure templates and policies + // are created before write aliases, to ensure templates are applied to the indices. Setup(template, ilm LoadMode) error } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 1fae6c6d8e3..13d1a201fcf 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -240,6 +240,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string return warn == "", warn } +// func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log := m.support.log @@ -267,16 +268,6 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { if policyCreated && templateComponent.enabled { templateComponent.overwrite = true } - - // create alias - if err := m.ilm.EnsureAlias(); err != nil { - if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists { - return err - } - log.Info("Write alias exists already") - } else { - log.Info("Write alias successfully generated.") - } } if templateComponent.load { @@ -298,6 +289,18 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Loaded index template.") } + if ilmComponent.load { + // ensure alias is created after the template is created + if err := m.ilm.EnsureAlias(); err != nil { + if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists { + return err + } + log.Info("Write alias exists already") + } else { + log.Info("Write alias successfully generated.") + } + } + return nil } diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 0c673ccfb28..836a6d11797 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -7,17 +7,20 @@ class IdxMgmt(object): def __init__(self, client, index): self._client = client - if index == "": - index == "mockbeat" - self._index = index + self._index = index if index != '' and index != '*' else 'mockbeat' def needs_init(self, s): return s == '' or s == '*' - def delete(self, index=""): - self.delete_index_and_alias(index=index) - self.delete_template(template=index) - self.delete_policy(policy=index) + def delete(self, indices=[]): + indices = list(filter(lambda x: x != '', indices)) + if not indices: + indices == [self._index] + for i in indices: + self.delete_index_and_alias(i) + self.delete_template(template=i) + for i in indices: + self.delete_policy(policy=i) def delete_index_and_alias(self, index=""): if self.needs_init(index): diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 8f556942adc..bcf6d7297b9 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -3,9 +3,9 @@ import os from nose.plugins.attrib import attr import unittest -import shutil -import datetime import logging +from nose.tools import raises +from elasticsearch import RequestError INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -28,17 +28,13 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -64,6 +60,26 @@ def test_setup_default(self): self.idxmgmt.assert_alias_created(self.alias_name) self.idxmgmt.assert_policy_created(self.policy_name) + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + @raises(RequestError) + def test_setup_default(self): + """ + Test setup --index-management with default config + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_policy_created(self.policy_name) + # try deleting policy needs to raise an error as it is in use + self.idxmgmt.delete_policy(self.policy_name) + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') def test_setup_template_disabled(self): diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index aede356fd79..c73934ca972 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -21,14 +21,10 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -169,17 +165,13 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def render_config(self, **kwargs): self.render_config_template( diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 03bf017ee06..084993b8850 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -128,10 +128,10 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.index_name]) def tearDown(self): - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -186,14 +186,12 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) def render_config(self, **kwargs): self.render_config_template(