Load index templates v2 by default when talking to ES 7.16 or ES 8.x (#…

…28538) ## What does this PR do? From now on Beats is going to load the new index templates to Elasticsearch 7.x and 8.x. The PR consists of a few test fixes, and I had found an issue in templates, that I will port to master. ## Why is it important? This way we can be forward compatible with Elasticsearch 8.x where legacy templates are no longer available.
elastic · Oct 21, 2021 · f0c5b73 · f0c5b73
1 parent c3a793e
commit f0c5b73
Show file tree

Hide file tree

Showing 30 changed files with 189 additions and 134 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -19,6 +19,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix panic with inline SSL when the certificate or key were small than 256 bytes. {pull}23820[23820]
 - Skip add_kubernetes_metadata processor when kubernetes metadata are already present {pull}27689[27689]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
+- Load index templates v2 (composable index templates) by default when talking to ES 7.16 or ES 8.x. Please note that you cannot load templates
+into Elasticsearch 7.8 or older with this default. To load templates to these ES version, set `setup.template.type` back
+to `legacy`. {pull}28538[28538]
 
 *Auditbeat*
 

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -1206,8 +1206,8 @@ output.elasticsearch:
 
 # Select the kind of index template. From Elasticsearch 7.8, it is possible to
 # use component templates. Available options: legacy, component, index.
-# By default auditbeat uses the legacy index templates.
-#setup.template.type: legacy
+# By default auditbeat uses index templates.
+#setup.template.type: index
 
 # Template name. By default the template name is "auditbeat-%{[agent.version]}"
 # The template name and pattern has to be set in case the Elasticsearch index pattern is modified.

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -2117,8 +2117,8 @@ output.elasticsearch:
 
 # Select the kind of index template. From Elasticsearch 7.8, it is possible to
 # use component templates. Available options: legacy, component, index.
-# By default filebeat uses the legacy index templates.
-#setup.template.type: legacy
+# By default filebeat uses index templates.
+#setup.template.type: index
 
 # Template name. By default the template name is "filebeat-%{[agent.version]}"
 # The template name and pattern has to be set in case the Elasticsearch index pattern is modified.

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -1398,8 +1398,8 @@ output.elasticsearch:
 
 # Select the kind of index template. From Elasticsearch 7.8, it is possible to
 # use component templates. Available options: legacy, component, index.
-# By default heartbeat uses the legacy index templates.
-#setup.template.type: legacy
+# By default heartbeat uses index templates.
+#setup.template.type: index
 
 # Template name. By default the template name is "heartbeat-%{[agent.version]}"
 # The template name and pattern has to be set in case the Elasticsearch index pattern is modified.

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -1148,8 +1148,8 @@ output.elasticsearch:
 
 # Select the kind of index template. From Elasticsearch 7.8, it is possible to
 # use component templates. Available options: legacy, component, index.
-# By default journalbeat uses the legacy index templates.
-#setup.template.type: legacy
+# By default journalbeat uses index templates.
+#setup.template.type: index
 
 # Template name. By default the template name is "journalbeat-%{[agent.version]}"
 # The template name and pattern has to be set in case the Elasticsearch index pattern is modified.

diff --git a/libbeat/_meta/config/setup.template.reference.yml.tmpl b/libbeat/_meta/config/setup.template.reference.yml.tmpl
@@ -9,8 +9,8 @@
 
 # Select the kind of index template. From Elasticsearch 7.8, it is possible to
 # use component templates. Available options: legacy, component, index.
-# By default {{.BeatName}} uses the legacy index templates.
-#setup.template.type: legacy
+# By default {{.BeatName}} uses index templates.
+#setup.template.type: index
 
 # Template name. By default the template name is "{{.BeatIndexPrefix}}-%{[agent.version]}"
 # The template name and pattern has to be set in case the Elasticsearch index pattern is modified.

diff --git a/libbeat/template/config.go b/libbeat/template/config.go
@@ -20,6 +20,7 @@ package template
 import (
 	"fmt"
 
+	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/v7/libbeat/mapping"
 )
 
@@ -69,18 +70,20 @@ func DefaultConfig() TemplateConfig {
 	return TemplateConfig{
 		Enabled:  true,
 		Fields:   "",
-		Type:     IndexTemplateLegacy,
+		Type:     IndexTemplateIndex,
 		Order:    1,
 		Priority: 150,
 	}
 }
 
 func (t *IndexTemplateType) Unpack(v string) error {
 	if v == "" {
-		*t = IndexTemplateLegacy
+		*t = IndexTemplateIndex
 		return nil
 	}
 
+	cfgwarn.Deprecate("8.0.0", "do not use setup.template.type, it is deprecated, data streams will be loaded automatically")
+
 	var tt IndexTemplateType
 	var ok bool
 	if tt, ok = templateTypes[v]; !ok {

diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go
@@ -223,7 +223,7 @@ func TestESLoader_Load(t *testing.T) {
 				// Fetch properties
 				tmpl := getTemplate(t, setup.client, setup.config.Name, setup.config.Type)
 				val, err := tmpl.GetValue("mappings.properties")
-				if data.properties == nil {
+				if data.properties == nil && setup.config.Type != IndexTemplateLegacy {
 					assert.Error(t, err)
 				} else {
 					require.NoError(t, err)

diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go
@@ -31,7 +31,6 @@ import (
 func TestFileLoader_Load(t *testing.T) {
 	ver := "7.0.0"
 	prefix := "mock"
-	order := 1
 	info := beat.Info{Version: ver, IndexPrefix: prefix}
 	tmplName := fmt.Sprintf("%s-%s", prefix, ver)
 
@@ -42,29 +41,35 @@ func TestFileLoader_Load(t *testing.T) {
 		"load minimal config info": {
 			body: common.MapStr{
 				"index_patterns": []string{"mock-7.0.0-*"},
-				"order":          order,
-				"settings":       common.MapStr{"index": nil}},
+				"priority":       150,
+				"template": common.MapStr{
+					"settings": common.MapStr{"index": nil}},
+			},
 		},
 		"load minimal config with index settings": {
 			settings: TemplateSettings{Index: common.MapStr{"code": "best_compression"}},
 			body: common.MapStr{
 				"index_patterns": []string{"mock-7.0.0-*"},
-				"order":          order,
-				"settings":       common.MapStr{"index": common.MapStr{"code": "best_compression"}}},
+				"priority":       150,
+				"template": common.MapStr{
+					"settings": common.MapStr{"index": common.MapStr{"code": "best_compression"}}},
+			},
 		},
 		"load minimal config with source settings": {
 			settings: TemplateSettings{Source: common.MapStr{"enabled": false}},
 			body: common.MapStr{
 				"index_patterns": []string{"mock-7.0.0-*"},
-				"order":          order,
-				"settings":       common.MapStr{"index": nil},
-				"mappings": common.MapStr{
-					"_source":           common.MapStr{"enabled": false},
-					"_meta":             common.MapStr{"beat": prefix, "version": ver},
-					"date_detection":    false,
-					"dynamic_templates": nil,
-					"properties":        nil,
-				}},
+				"priority":       150,
+				"template": common.MapStr{
+					"settings": common.MapStr{"index": nil},
+					"mappings": common.MapStr{
+						"_source":           common.MapStr{"enabled": false},
+						"_meta":             common.MapStr{"beat": prefix, "version": ver},
+						"date_detection":    false,
+						"dynamic_templates": nil,
+						"properties":        nil,
+					}},
+			},
 		},
 	} {
 		t.Run(name, func(t *testing.T) {

diff --git a/libbeat/template/template.go b/libbeat/template/template.go
@@ -200,13 +200,6 @@ func (t *Template) LoadMinimal() (common.MapStr, error) {
 		return nil, fmt.Errorf("unknown template type %v", t.templateType)
 	}
 
-	if t.config.Settings.Source != nil {
-		m["mappings"] = buildMappings(
-			t.beatVersion, t.esVersion, t.beatName,
-			nil, nil,
-			common.MapStr(t.config.Settings.Source))
-	}
-
 	return m, nil
 }
 
@@ -218,22 +211,34 @@ func (t *Template) loadMinimalLegacy() common.MapStr {
 		"settings": common.MapStr{
 			"index": t.config.Settings.Index,
 		},
+		"mappings": buildMappings(
+			t.beatVersion, t.esVersion, t.beatName,
+			nil, nil,
+			common.MapStr(t.config.Settings.Source)),
 	}
 }
 
 func (t *Template) loadMinimalComponent() common.MapStr {
+	templ := common.MapStr{}
+	if t.config.Settings.Source != nil {
+		templ["mappings"] = buildMappings(
+			t.beatVersion, t.esVersion, t.beatName,
+			nil, nil,
+			common.MapStr(t.config.Settings.Source))
+	}
+	templ["settings"] = common.MapStr{
+		"index": t.config.Settings.Index,
+	}
 	return common.MapStr{
-		"template": common.MapStr{
-			"settings": common.MapStr{
-				"index": t.config.Settings.Index,
-			},
-		},
+		"template": templ,
 	}
 }
 
 func (t *Template) loadMinimalIndex() common.MapStr {
+	keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern())
 	m := t.loadMinimalComponent()
 	m["priority"] = t.priority
+	m[keyPattern] = patterns
 	return m
 }
 

diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go
@@ -110,7 +110,9 @@ func TestTemplate(t *testing.T) {
 	currentVersion := getVersion("")
 
 	t.Run("for ES 6.x", func(t *testing.T) {
-		template := createTestTemplate(t, currentVersion, "6.4.0", DefaultConfig())
+		cfg := DefaultConfig()
+		cfg.Type = IndexTemplateLegacy
+		template := createTestTemplate(t, currentVersion, "6.4.0", cfg)
 		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
 		template.Assert("order", 1)
 		template.Assert("mappings.doc._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
@@ -120,17 +122,15 @@ func TestTemplate(t *testing.T) {
 	t.Run("for ES 7.x", func(t *testing.T) {
 		template := createTestTemplate(t, currentVersion, "7.2.0", DefaultConfig())
 		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
-		template.Assert("order", 1)
-		template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
-		template.Assert("settings.index.max_docvalue_fields_search", 200)
+		template.Assert("template.mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+		template.Assert("template.settings.index.max_docvalue_fields_search", 200)
 	})
 
 	t.Run("for ES 8.x", func(t *testing.T) {
 		template := createTestTemplate(t, currentVersion, "8.0.0", DefaultConfig())
 		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
-		template.Assert("order", 1)
-		template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
-		template.Assert("settings.index.max_docvalue_fields_search", 200)
+		template.Assert("template.mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+		template.Assert("template.settings.index.max_docvalue_fields_search", 200)
 	})
 }
 

diff --git a/libbeat/tests/files/template.json b/libbeat/tests/files/template.json
@@ -1,19 +1,21 @@
 {
     "index_patterns": ["bla"],
-    "settings": {
-        "number_of_shards": 1
-    },
-    "mappings": {
-        "_source": {
-            "enabled": false
+    "template": {
+        "settings": {
+            "number_of_shards": 1
         },
-        "properties": {
-            "host_name": {
-                "type": "keyword"
+        "mappings": {
+            "_source": {
+                "enabled": false
             },
-            "created_at": {
-                "type": "date",
-                "format": "EEE MMM dd HH:mm:ss Z YYYY"
+            "properties": {
+                "host_name": {
+                    "type": "keyword"
+                },
+                "created_at": {
+                    "type": "date",
+                    "format": "EEE MMM dd HH:mm:ss Z YYYY"
+                }
             }
         }
     }

diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py
@@ -52,7 +52,12 @@ def test_export_template(self):
         """
         output = self.run_export_cmd("template")
         js = json.loads(output)
-        assert "index_patterns" in js and "mappings" in js
+        assert "index_patterns" in js
+        assert "template" in js
+        assert "priority" in js
+        assert "order" not in js
+        assert "mappings" in js["template"]
+        assert "settings" in js["template"]
 
     def test_export_index_pattern(self):
         """

diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py
@@ -37,7 +37,7 @@ def delete_template(self, template=""):
             template = self._index
 
         try:
-            self._client.transport.perform_request('DELETE', "/_template/" + template + "*")
+            self._client.transport.perform_request('DELETE', "/_index_template/" + template + "*")
         except NotFoundError:
             pass
 
@@ -54,7 +54,7 @@ def delete_policy(self, policy):
 
     def assert_index_template_not_loaded(self, template):
         with pytest.raises(NotFoundError):
-            self._client.transport.perform_request('GET', '/_template/' + template)
+            self._client.transport.perform_request('GET', '/_index_template/' + template)
 
     def assert_legacy_index_template_loaded(self, template):
         resp = self._client.transport.perform_request('GET', '/_template/' + template)
@@ -69,6 +69,16 @@ def assert_index_template_loaded(self, template):
                 found = True
         assert found
 
+    def assert_ilm_index_template_loaded(self, template, policy, alias):
+        resp = self._client.transport.perform_request('GET', '/_index_template/' + template)
+        found = False
+        for index_template in resp['index_templates']:
+            if index_template['name'] == template:
+                found = True
+                assert index_template['index_template']['template']['settings']["index"]["lifecycle"]["name"] == policy
+                assert index_template['index_template']['template']['settings']["index"]["lifecycle"]["rollover_alias"] == alias
+        assert found
+
     def assert_component_template_loaded(self, template):
         resp = self._client.transport.perform_request('GET', '/_component_template/' + template)
         found = False
@@ -84,9 +94,12 @@ def assert_ilm_template_loaded(self, template, policy, alias):
         assert resp[template]["settings"]["index"]["lifecycle"]["rollover_alias"] == alias
 
     def assert_index_template_index_pattern(self, template, index_pattern):
-        resp = self._client.transport.perform_request('GET', '/_template/' + template)
-        assert template in resp
-        assert resp[template]["index_patterns"] == index_pattern
+        resp = self._client.transport.perform_request('GET', '/_index_template/' + template)
+        for index_template in resp['index_templates']:
+            if index_template['name'] == template:
+                assert index_pattern == index_template['index_template']['index_patterns']
+                found = True
+        assert found
 
     def assert_alias_not_created(self, alias):
         resp = self._client.transport.perform_request('GET', '/_alias')