Update beats framework to 1c1b7d7

NOTICE.txt

@@ -264,7 +264,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 675f2b000b067ad9fb2ee2fbe2bb3d98ecfab30c
+Revision: 1c1b7d7427a422ec9da0707b1830e9cb03837266
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/dev-tools/cmd/dashboards/export_dashboards.go

@@ -18,132 +18,27 @@
 package main
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"flag"
-	"fmt"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
+	"time"
 
-	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/dashboards"
 	"github.com/elastic/beats/libbeat/kibana"
 )
 
-var exportAPI = "/api/kibana/dashboards/export"
-
-type manifest struct {
-	Dashboards []map[string]string `config:"dashboards"`
-}
-
-func makeURL(url, path string, params url.Values) string {
-	if len(params) == 0 {
-		return url + path
-	}
-
-	return strings.Join([]string{url, path, "?", params.Encode()}, "")
-}
-
-func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error {
-	params := url.Values{}
-
-	params.Add("dashboard", dashboard)
-
-	if spaceID != "" {
-		exportAPI = path.Join("/s", spaceID, exportAPI)
-	}
-	fullURL := makeURL(conn, exportAPI, params)
-	if !quiet {
-		log.Printf("Calling HTTP GET %v\n", fullURL)
-	}
-
-	req, err := http.NewRequest("GET", fullURL, nil)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("GET HTTP request fails with: %v", err)
-	}
-
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("fail to read response %s", err)
-	}
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("HTTP GET %s fails with %s, %s", fullURL, resp.Status, body)
-	}
-
-	data, err := kibana.RemoveIndexPattern(body)
-	if err != nil {
-		return fmt.Errorf("fail to extract the index pattern: %v", err)
-	}
-
-	objects := data["objects"].([]interface{})
-	for _, obj := range objects {
-		o := obj.(common.MapStr)
-
-		// All fields are optional, so errors are not catched
-		decodeValue(o, "attributes.uiStateJSON")
-		decodeValue(o, "attributes.visState")
-		decodeValue(o, "attributes.optionsJSON")
-		decodeValue(o, "attributes.panelsJSON")
-		decodeValue(o, "attributes.kibanaSavedObjectMeta.searchSourceJSON")
-	}
-
-	data["objects"] = objects
-
-	// Create all missing directories
-	err = os.MkdirAll(filepath.Dir(out), 0755)
-	if err != nil {
-		return err
-	}
-
-	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0644)
-	if !quiet {
-		log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out)
-	}
-	return err
-}
-
-func decodeValue(data common.MapStr, key string) error {
-	v, err := data.GetValue(key)
-	if err != nil {
-		return err
-	}
-	s := v.(string)
-	var d interface{}
-	err = json.Unmarshal([]byte(s), &d)
-	if err != nil {
-		return fmt.Errorf("error decoding %s: %v", key, err)
-	}
-
-	data.Put(key, d)
-	return nil
-}
-
-func ReadManifest(file string) ([]map[string]string, error) {
-	cfg, err := common.LoadFile(file)
-	if err != nil {
-		return nil, fmt.Errorf("error reading manifest file: %v", err)
-	}
-
-	var manifest manifest
-	err = cfg.Unpack(&manifest)
-	if err != nil {
-		return nil, fmt.Errorf("error unpacking manifest: %v", err)
-	}
-	return manifest.Dashboards, nil
-}
+var (
+	indexPattern = false
+	quiet        = false
+)
 
-var indexPattern = false
-var quiet = false
+const (
+	kibanaTimeout = 90 * time.Second
+)
 
 func main() {
 	kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
@@ -157,42 +52,57 @@ func main() {
 	flag.Parse()
 	log.SetFlags(0)
 
-	transCfg := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates
+	u, err := url.Parse(*kibanaURL)
+	if err != nil {
+		log.Fatalf("Error parsing Kibana URL: %v", err)
 	}
 
-	client := &http.Client{Transport: transCfg}
+	client, err := kibana.NewClientWithConfig(&kibana.ClientConfig{
+		Protocol: u.Scheme,
+		Host:     u.Host,
+		SpaceID:  *spaceID,
+		Timeout:  kibanaTimeout,
+	})
+	if err != nil {
+		log.Fatalf("Error while connecting to Kibana: %+v", err)
+	}
 
 	if len(*ymlFile) == 0 && len(*dashboard) == 0 {
 		flag.Usage()
 		log.Fatalf("Please specify a dashboard ID (-dashboard) or a manifest file (-yml)")
 	}
 
 	if len(*ymlFile) > 0 {
-		dashboards, err := ReadManifest(*ymlFile)
-		if err != nil {
-			log.Fatalf("%s", err)
-		}
-
-		for _, dashboard := range dashboards {
-			log.Printf("id=%s, name=%s\n", dashboard["id"], dashboard["file"])
-			directory := filepath.Join(filepath.Dir(*ymlFile), "_meta/kibana/6/dashboard")
-			err := os.MkdirAll(directory, 0755)
-			if err != nil {
-				log.Fatalf("fail to create directory %s: %v", directory, err)
-			}
-			err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"]))
+		results, info, err := dashboards.ExportAllFromYml(client, *ymlFile)
+		for i, r := range results {
+			log.Printf("id=%s, name=%s\n", info.Dashboards[i].ID, info.Dashboards[i].File)
+			r = dashboards.DecodeExported(r)
+			err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(*ymlFile), client.GetVersion())
 			if err != nil {
-				log.Fatalf("fail to export the dashboards: %s", err)
+				log.Fatalf("failed to export the dashboards: %s", err)
 			}
 		}
 		os.Exit(0)
 	}
 
 	if len(*dashboard) > 0 {
-		err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput)
+		result, err := dashboards.Export(client, *dashboard)
+		if err != nil {
+			log.Fatalf("Failed to export the dashboard: %s", err)
+		}
+		result = dashboards.DecodeExported(result)
+		bytes, err := json.Marshal(result)
+		if err != nil {
+			log.Fatalf("Failed to save the dashboard: %s", err)
+		}
+
+		err = ioutil.WriteFile(*fileOutput, bytes, 0644)
 		if err != nil {
-			log.Fatalf("fail to export the dashboards: %s", err)
+			log.Fatalf("Failed to save the dashboard: %s", err)
+
+		}
+		if !quiet {
+			log.Printf("The dashboard %s was exported under the %s file\n", *dashboard, *fileOutput)
 		}
 	}
 }

_beats/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -79,7 +79,7 @@ func main() {
 
 		file, err := indexPattern.Generate()
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("ERROR: %s", err)
 		}
 
 		// Log output file location.

_beats/dev-tools/ecs-migration.yml

@@ -105,6 +105,56 @@
   alias: true
   copy_to: false
 
+- from: system.auth.hostname
+  to: host.hostname
+  alias: true
+  copy_to: false
+
+- from: system.auth.pid
+  to: process.pid
+  alias: true
+  copy_to: false
+
+- from: system.auth.groupadd.gid
+  to: group.id
+  alias: true
+  copy_to: false
+
+- from: system.auth.useradd.uid
+  to: user.id
+  alias: true
+  copy_to: false
+
+- from: system.auth.useradd.user
+  to: user.name
+  alias: true
+  copy_to: false
+
+- from: system.auth.ssh.event
+  to: event.action
+  alias: true
+  copy_to: false
+
+- from: system.auth.program
+  to: process.name
+  alias: true
+  copy_to: false
+
+- from: system.auth.ssh.ip
+  to: source.ip
+  alias: true
+  copy_to: false
+
+- from: system.auth.ssh.port
+  to: source.port
+  alias: true
+  copy_to: false
+
+- from: system.auth.ssh.geoip.*
+  to: source.geo.*
+  alias: true
+  copy_to: false
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain
@@ -252,3 +302,80 @@
   to: source.geo.region_iso_code
   alias: true
   copy_to: false
+
+- from: nginx.access.remote_ip_list
+  to: network.forwarded_ip
+  alias: true
+  copy_to: false
+
+- from: nginx.access.user_name
+  to: user.name
+  alias: true
+  copy_to: false
+
+- from: nginx.access.url
+  to: url.original
+  alias: true
+  copy_to: false
+
+- from: nginx.access.agent
+  to: user_agent.original
+  alias: true
+  copy_to: false
+
+ # Note: `http` is not officially in ECS yet
+
+- from: nginx.access.response_code
+  to: http.response.status_code
+  alias: true
+  copy_to: false
+
+- from: nginx.access.referrer
+  to: http.request.referrer
+  alias: true
+  copy_to: false
+
+- from: nginx.access.method
+  to: http.request.method
+  alias: true
+  copy_to: false
+
+- from: nginx.access.http_version
+  to: http.version
+  alias: true
+  copy_to: false
+
+- from: nginx.access.geoip.continent_name
+  to: source.geo.continent_name
+  alias: true
+  copy_to: false
+
+- from: nginx.access.geoip.country_iso_code
+  to: source.geo.country_iso_code
+  alias: true
+  copy_to: false
+
+- from: nginx.access.geoip.location
+  to: source.geo.location
+  alias: true
+  copy_to: false
+
+- from: nginx.access.geoip.region_name
+  to: source.geo.region_name
+  alias: true
+  copy_to: false
+
+- from: nginx.access.geoip.city_name
+  to: source.geo.city_name
+  alias: true
+  copy_to: false
+
+- from: nginx.access.geoip.region_iso_code
+  to: source.geo.region_iso_code
+  alias: true
+  copy_to: false
+
+- from: nginx.access.agent
+  to: user_agent.original
+  alias: true
+  copy_to: false