From b054f080c871dadade8e1a6a72a7966f551d9ed4 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 31 Jan 2019 10:45:20 +0100 Subject: [PATCH] Clean up Kibana, remove Elasticsearch loading and 5.x version (#10451) In 5.x the dashboards were loaded through Elasticsearch and not Kibana. For 6.x and 7.x dashboards are loaded through the Kibana API. The old code is removed in this PR. Also some code is cleaned up around talking to 5.x Kibana APIs. --- libbeat/cmd/instance/beat.go | 7 +- libbeat/dashboards/dashboards.go | 87 +------ libbeat/dashboards/es_loader.go | 335 --------------------------- libbeat/dashboards/es_loader_test.go | 99 -------- libbeat/kibana/client.go | 22 +- 5 files changed, 11 insertions(+), 539 deletions(-) delete mode 100644 libbeat/dashboards/es_loader.go delete mode 100644 libbeat/dashboards/es_loader_test.go diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 2ab590dc900..d02c12e6185 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -705,13 +705,8 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { } if b.Config.Dashboards.Enabled() { - var esConfig *common.Config - - if b.Config.Output.Name() == "elasticsearch" { - esConfig = b.Config.Output.Config() - } err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""), - b.Config.Kibana, esConfig, b.Config.Dashboards, nil) + b.Config.Kibana, b.Config.Dashboards, nil) if err != nil { return errw.Wrap(err, "Error importing Kibana dashboards") } diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 833c7a1ebe8..79d664c3db8 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -28,23 +28,11 @@ import ( "github.com/elastic/beats/libbeat/common" ) -type importMethod uint8 - -// check import route -const ( - importNone importMethod = iota - importViaKibana - importViaES -) - // ImportDashboards tries to import the kibana dashboards. -// If the Elastic Stack is at version 6.0+, the dashboards should be installed -// via the kibana dashboard loader plugin. For older versions of the Elastic Stack -// we write the dashboards directly into the .kibana index. func ImportDashboards( ctx context.Context, beatName, hostname, homePath string, - kibanaConfig, esConfig, dashboardsConfig *common.Config, + kibanaConfig, dashboardsConfig *common.Config, msgOutputter MessageOutputter, ) error { if dashboardsConfig == nil || !dashboardsConfig.Enabled() { @@ -65,61 +53,12 @@ func ImportDashboards( kibanaConfig = common.NewConfig() } - if esConfig.Enabled() { - username, _ := esConfig.String("username", -1) - password, _ := esConfig.String("password", -1) - - if !kibanaConfig.HasField("username") && username != "" { - kibanaConfig.SetString("username", -1, username) - } - if !kibanaConfig.HasField("password") && password != "" { - kibanaConfig.SetString("password", -1, password) - } - } - - var esLoader *ElasticsearchLoader - - importVia := importNone - useKibana := importViaKibana if !kibanaConfig.Enabled() { - useKibana = importNone + return errors.New("kibana configuration missing for loading dashboards.") } - requiresKibana := dashConfig.AlwaysKibana || !esConfig.Enabled() - if requiresKibana { - importVia = useKibana - } else { - // Check import route via elasticsearch version. If Elasticsearch major - // version is >6, we assume Kibana also being at versions >6.0. In this - // case dashboards will be imported using the new kibana dashboard loader - // plugin. - // XXX(urso): Why do we test the Elasticsearch version? If kibana is - // configured, why not test the kibana version and plugin - // availability first? - esLoader, err = NewElasticsearchLoader(esConfig, &dashConfig, msgOutputter) - if err != nil { - return fmt.Errorf("fail to create the Elasticsearch loader: %v", err) - } - defer esLoader.Close() - - esLoader.statusMsg("Elasticsearch URL %v", esLoader.client.Connection.URL) - - if esLoader.version.Major < 6 { - importVia = importViaES - } else { - importVia = useKibana - } - } + return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter) - // Try to import dashboards. - switch importVia { - case importViaES: - return ImportDashboardsViaElasticsearch(esLoader) - case importViaKibana: - return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter) - default: - return errors.New("Elasticsearch or Kibana configuration missing for loading dashboards.") - } } func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config, @@ -159,26 +98,6 @@ func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader) error { return nil } -func ImportDashboardsViaElasticsearch(esLoader *ElasticsearchLoader) error { - - if err := esLoader.CreateKibanaIndex(); err != nil { - return fmt.Errorf("fail to create the kibana index: %v", err) - } - - version, _ := common.NewVersion("5.0.0") - - importer, err := NewImporter(*version, esLoader.config, esLoader) - if err != nil { - return fmt.Errorf("fail to create an Elasticsearch importer for loading the dashboards: %v", err) - } - - if err := importer.Import(); err != nil { - return fmt.Errorf("fail to import the dashboards in Elasticsearch: %v", err) - } - - return nil -} - func isKibanaAPIavailable(version common.Version) bool { return (version.Major == 5 && version.Minor >= 6) || version.Major >= 6 } diff --git a/libbeat/dashboards/es_loader.go b/libbeat/dashboards/es_loader.go deleted file mode 100644 index 4787bb3faf5..00000000000 --- a/libbeat/dashboards/es_loader.go +++ /dev/null @@ -1,335 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package dashboards - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "path" - "path/filepath" - "strings" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/libbeat/outputs/elasticsearch" -) - -type ElasticsearchLoader struct { - client *elasticsearch.Client - config *Config - version common.Version - msgOutputter MessageOutputter -} - -func NewElasticsearchLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter) (*ElasticsearchLoader, error) { - if cfg == nil || !cfg.Enabled() { - return nil, fmt.Errorf("Elasticsearch output is not configured/enabled") - } - - esClient, err := elasticsearch.NewConnectedClient(cfg) - if err != nil { - return nil, fmt.Errorf("Error creating Elasticsearch client: %v", err) - } - - version := esClient.GetVersion() - if !version.IsValid() { - return nil, errors.New("No valid Elasticsearch version available") - } - - loader := ElasticsearchLoader{ - client: esClient, - config: dashboardsConfig, - version: version, - msgOutputter: msgOutputter, - } - - loader.statusMsg("Initialize the Elasticsearch %s loader", version.String()) - - return &loader, nil -} - -// CreateKibanaIndex creates the kibana index if it doesn't exists and sets -// some index properties which are needed as a workaround for: -// https://github.com/elastic/beats-dashboards/issues/94 -func (loader ElasticsearchLoader) CreateKibanaIndex() error { - status, err := loader.client.IndexExists(loader.config.KibanaIndex) - - if err != nil { - if status != 404 { - return err - } - - _, _, err = loader.client.CreateIndex(loader.config.KibanaIndex, nil) - if err != nil { - return fmt.Errorf("Failed to create index: %v", err) - } - - _, _, err = loader.client.CreateIndex(loader.config.KibanaIndex+"/_mapping/search", - common.MapStr{ - "search": common.MapStr{ - "properties": common.MapStr{ - "hits": common.MapStr{ - "type": "integer", - }, - "version": common.MapStr{ - "type": "integer", - }, - }, - }, - }) - if err != nil { - return fmt.Errorf("Failed to set the mapping: %v", err) - } - } - - return nil -} - -func (loader ElasticsearchLoader) ImportIndex(file string) error { - reader, err := ioutil.ReadFile(file) - if err != nil { - return err - } - var indexContent common.MapStr - err = json.Unmarshal(reader, &indexContent) - if err != nil { - return fmt.Errorf("fail to unmarshal index content: %v", err) - } - - indexName, ok := indexContent["title"].(string) - if !ok { - return fmt.Errorf("Missing title in the index-pattern file at %s", file) - } - - if loader.config.Index != "" { - // change index pattern name - loader.statusMsg("Change index in index-pattern %s", indexName) - indexContent["title"] = loader.config.Index - } - - path := "/" + loader.config.KibanaIndex + "/index-pattern/" + indexName - - if _, err = loader.client.LoadJSON(path, indexContent); err != nil { - return err - } - - return nil -} - -func (loader ElasticsearchLoader) importJSONFile(fileType string, file string) error { - path := "/" + loader.config.KibanaIndex + "/" + fileType - - reader, err := ioutil.ReadFile(file) - if err != nil { - return fmt.Errorf("Failed to read %s. Error: %s", file, err) - } - var jsonContent map[string]interface{} - err = json.Unmarshal(reader, &jsonContent) - if err != nil { - return fmt.Errorf("fail to unmarshal json file: %v", err) - } - - fileBase := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file)) - - body, err := loader.client.LoadJSON(path+"/"+fileBase, jsonContent) - if err != nil { - return fmt.Errorf("Failed to load %s under %s/%s: %s. Response body: %s", file, path, fileBase, err, body) - } - - return nil -} - -func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err error) { - // directory with the dashboards - dir := filepath.Dir(file) - - // main directory with dashboard, search, visualizations directories - mainDir := filepath.Dir(dir) - - reader, err := ioutil.ReadFile(file) - if err != nil { - return - } - type record struct { - Title string `json:"title"` - PanelsJSON string `json:"panelsJSON"` - } - type panel struct { - ID string `json:"id"` - Type string `json:"type"` - } - - var jsonContent record - err = json.Unmarshal(reader, &jsonContent) - if err != nil { - return fmt.Errorf("fail to unmarshal json content: %v", err) - } - - var widgets []panel - err = json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets) - if err != nil { - return fmt.Errorf("fail to unmarshal panels content: %v", err) - } - - for _, widget := range widgets { - if widget.Type == "visualization" { - err = loader.importVisualization(path.Join(mainDir, "visualization", widget.ID+".json")) - if err != nil { - return err - } - } else if widget.Type == "search" { - err = loader.importSearch(path.Join(mainDir, "search", widget.ID+".json")) - if err != nil { - return err - } - } else { - loader.statusMsg("Widgets: %v", widgets) - return fmt.Errorf("Unknown panel type %s in %s", widget.Type, file) - } - } - return -} - -func (loader ElasticsearchLoader) importVisualization(file string) error { - loader.statusMsg("Import visualization %s", file) - reader, err := ioutil.ReadFile(file) - if err != nil { - return err - } - var vizContent common.MapStr - err = json.Unmarshal(reader, &vizContent) - if err != nil { - return fmt.Errorf("fail to unmarshal visualization content %s: %v", file, err) - } - - if loader.config.Index != "" { - if savedObject, ok := vizContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok { - vizContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject) - } - - if visState, ok := vizContent["visState"].(string); ok { - vizContent["visState"] = ReplaceIndexInVisState(loader.config.Index, visState) - } - } - - vizName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file)) - path := "/" + loader.config.KibanaIndex + "/visualization/" + vizName - if _, err := loader.client.LoadJSON(path, vizContent); err != nil { - return err - } - - return loader.importSearchFromVisualization(file) -} - -func (loader ElasticsearchLoader) importSearch(file string) error { - reader, err := ioutil.ReadFile(file) - if err != nil { - return err - } - searchName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file)) - - var searchContent common.MapStr - err = json.Unmarshal(reader, &searchContent) - if err != nil { - return fmt.Errorf("fail to unmarshal search content %s: %v", searchName, err) - } - - if loader.config.Index != "" { - - // change index pattern name - if savedObject, ok := searchContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok { - - searchContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject) - - } - } - - path := "/" + loader.config.KibanaIndex + "/search/" + searchName - loader.statusMsg("Import search %s", file) - - if _, err = loader.client.LoadJSON(path, searchContent); err != nil { - return err - } - - return nil -} - -func (loader ElasticsearchLoader) importSearchFromVisualization(file string) error { - type record struct { - Title string `json:"title"` - SavedSearchID string `json:"savedSearchId"` - } - - reader, err := ioutil.ReadFile(file) - if err != nil { - return nil - } - - var jsonContent record - err = json.Unmarshal(reader, &jsonContent) - if err != nil { - return fmt.Errorf("fail to unmarshal the search content: %v", err) - } - - id := jsonContent.SavedSearchID - if len(id) == 0 { - // no search used - return nil - } - - // directory with the visualizations - dir := filepath.Dir(file) - - // main directory - mainDir := filepath.Dir(dir) - - searchFile := path.Join(mainDir, "search", id+".json") - - if searchFile != "" { - // visualization depends on search - if err := loader.importSearch(searchFile); err != nil { - return err - } - } - return nil -} - -func (loader ElasticsearchLoader) ImportDashboard(file string) error { - /* load dashboard */ - err := loader.importJSONFile("dashboard", file) - if err != nil { - return err - } - - /* load the visualizations and searches that depend on the dashboard */ - return loader.importPanelsFromDashboard(file) -} - -func (loader ElasticsearchLoader) Close() error { - return loader.client.Close() -} - -func (loader ElasticsearchLoader) statusMsg(msg string, a ...interface{}) { - if loader.msgOutputter != nil { - loader.msgOutputter(msg, a...) - } else { - logp.Debug("dashboards", msg, a...) - } -} diff --git a/libbeat/dashboards/es_loader_test.go b/libbeat/dashboards/es_loader_test.go deleted file mode 100644 index 29f5684da90..00000000000 --- a/libbeat/dashboards/es_loader_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build integration - -package dashboards - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/libbeat/outputs/elasticsearch/estest" -) - -func TestImporter(t *testing.T) { - logp.TestingSetup() - - dashboardsConfig := Config{ - KibanaIndex: ".kibana-test", - File: "testdata/testbeat-dashboards.zip", - Beat: "testbeat", - } - - client := estest.GetTestingElasticsearch(t) - major := client.GetVersion().Major - - if major == 6 || major == 7 { - t.Skip("Skipping tests for Elasticsearch 6.x releases") - } - - loader := ElasticsearchLoader{ - client: client, - config: &dashboardsConfig, - } - - err := loader.CreateKibanaIndex() - - assert.NoError(t, err) - - version, _ := common.NewVersion("5.0.0") - - imp, err := NewImporter(*version, &dashboardsConfig, loader) - assert.NoError(t, err) - - err = imp.Import() - assert.NoError(t, err) - - status, _, _ := client.Request("GET", "/.kibana-test/dashboard/1e4389f0-e871-11e6-911d-3f8ed6f72700", "", nil, nil) - assert.Equal(t, 200, status) -} - -func TestImporterEmptyBeat(t *testing.T) { - logp.TestingSetup() - - dashboardsConfig := Config{ - KibanaIndex: ".kibana-test-nobeat", - File: "testdata/testbeat-dashboards.zip", - Beat: "", - } - - client := estest.GetTestingElasticsearch(t) - major := client.GetVersion().Major - if major == 6 || major == 7 { - t.Skip("Skipping tests for Elasticsearch 6.x releases") - } - - loader := ElasticsearchLoader{ - client: client, - config: &dashboardsConfig, - } - - version, _ := common.NewVersion("5.0.0") - - imp, err := NewImporter(*version, &dashboardsConfig, loader) - assert.NoError(t, err) - - err = imp.Import() - assert.NoError(t, err) - - status, _, _ := client.Request("GET", "/.kibana-test-nobeat/dashboard/1e4389f0-e871-11e6-911d-3f8ed6f72700", "", nil, nil) - assert.Equal(t, 200, status) -} diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index d759ad432fb..e28bfb8d4fa 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -226,23 +226,15 @@ func (client *Client) readVersion() error { var kibanaVersion kibanaVersionResponse err = json.Unmarshal(result, &kibanaVersion) if err != nil { - var kibanaVersion5x kibanaVersionResponse5x - - // The response returned by /api/status is different in Kibana 5.x than in Kibana 6.x - err5x := json.Unmarshal(result, &kibanaVersion5x) - if err5x != nil { + return fmt.Errorf("fail to unmarshal the response from GET %s/api/status. Response: %s. Kibana status api returns: %v", + client.Connection.URL, truncateString(result), err) + } - return fmt.Errorf("fail to unmarshal the response from GET %s/api/status. Response: %s. Kibana 5.x status api returns: %v. Kibana 6.x status api returns: %v", - client.Connection.URL, truncateString(result), err5x, err) - } - versionString = kibanaVersion5x.Version - } else { - versionString = kibanaVersion.Version.Number + versionString = kibanaVersion.Version.Number - if kibanaVersion.Version.Snapshot { - // needed for the tests - versionString += "-SNAPSHOT" - } + if kibanaVersion.Version.Snapshot { + // needed for the tests + versionString += "-SNAPSHOT" } version, err := common.NewVersion(versionString)