From a13e8f7576ee9d056f516a87b8aa54b35c9dd050 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 9 Jun 2020 07:31:53 -0700 Subject: [PATCH] Encode API key as base64 in common code (#18945) (#19064) * Encode API key as base64 in common code * Adding comment on API key field * Adding CHANGELOG entries * Adding test * Base64-encode API key in constructor * Move encodedAPIKey field to Connection * Update doc on `api_key` setting value * Adding API key format to setting section * Compute entire API key auth header value up front --- CHANGELOG-developer.next.asciidoc | 1 + CHANGELOG.next.asciidoc | 1 + libbeat/esleg/eslegclient/connection.go | 22 +++++-- libbeat/esleg/eslegclient/connection_test.go | 66 +++++++++++++++++++ libbeat/outputs/elasticsearch/client.go | 3 +- .../elasticsearch/docs/elasticsearch.asciidoc | 9 +-- 6 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 libbeat/esleg/eslegclient/connection_test.go diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 13d0a14ccb0..5987ce1a1f2 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -42,6 +42,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. Your magefile.go will require a change to adapt the devtool API. See the pull request for more details. {pull}18148[18148] - Introduce APM libbeat instrumentation. `Publish` method on `Client` interface now takes a Context as first argument. {pull}17938[17938] +- The Elasticsearch client settings expect the API key to be raw (not base64-encoded). {issue}18939[18939] {pull}18945[18945] ==== Bugfixes diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 96995e30551..d23ee3039cc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -124,6 +124,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix regression in `add_kubernetes_metadata`, so configured `indexers` and `matchers` are used if defaults are not disabled. {issue}18481[18481] {pull}18818[18818] - Fix potential race condition in fingerprint processor. {pull}18738[18738] - Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916] +- The `monitoring.elasticsearch.api_key` value is correctly base64-encoded before being sent to the monitoring Elasticsearch cluster. {issue}18939[18939] {pull}18945[18945] *Auditbeat* diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 46d4840cda8..3802ae1cb0e 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -18,6 +18,7 @@ package eslegclient import ( + "encoding/base64" "encoding/json" "fmt" "io" @@ -48,8 +49,9 @@ type Connection struct { Encoder BodyEncoder HTTP esHTTPClient - version common.Version - log *logp.Logger + apiKeyAuthHeader string // Authorization HTTP request header with base64-encoded API key + version common.Version + log *logp.Logger } // ConnectionSettings are the settings needed for a Connection @@ -60,7 +62,7 @@ type ConnectionSettings struct { Username string Password string - APIKey string + APIKey string // Raw API key, NOT base64-encoded Headers map[string]string TLS *tlscommon.TLSConfig @@ -157,12 +159,18 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { logp.Info("kerberos client created") } - return &Connection{ + conn := Connection{ ConnectionSettings: s, HTTP: httpClient, Encoder: encoder, log: logp.NewLogger("esclientleg"), - }, nil + } + + if s.APIKey != "" { + conn.apiKeyAuthHeader = "ApiKey " + base64.StdEncoding.EncodeToString([]byte(s.APIKey)) + } + + return &conn, nil } func settingsWithDefaults(s ConnectionSettings) ConnectionSettings { @@ -435,8 +443,8 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) req.SetBasicAuth(conn.Username, conn.Password) } - if conn.APIKey != "" { - req.Header.Add("Authorization", "ApiKey "+conn.APIKey) + if conn.apiKeyAuthHeader != "" { + req.Header.Add("Authorization", conn.apiKeyAuthHeader) } for name, value := range conn.Headers { diff --git a/libbeat/esleg/eslegclient/connection_test.go b/libbeat/esleg/eslegclient/connection_test.go new file mode 100644 index 00000000000..72b78f27e1d --- /dev/null +++ b/libbeat/esleg/eslegclient/connection_test.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package eslegclient + +import ( + "bufio" + "bytes" + "encoding/base64" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAPIKeyEncoding(t *testing.T) { + apiKey := "foobar" + encoded := base64.StdEncoding.EncodeToString([]byte(apiKey)) + + conn, err := NewConnection(ConnectionSettings{ + APIKey: apiKey, + }) + require.NoError(t, err) + + httpClient := newMockClient() + conn.HTTP = httpClient + + req, err := http.NewRequest("GET", "http://fakehost/some/path", nil) + require.NoError(t, err) + + _, _, err = conn.execHTTPRequest(req) + require.NoError(t, err) + + require.Equal(t, "ApiKey "+encoded, httpClient.Req.Header.Get("Authorization")) +} + +type mockClient struct { + Req *http.Request +} + +func (c *mockClient) Do(req *http.Request) (*http.Response, error) { + c.Req = req + + r := bytes.NewReader([]byte("HTTP/1.1 200 OK\n\nHello, world")) + return http.ReadResponse(bufio.NewReader(r), req) +} + +func (c *mockClient) CloseIdleConnections() {} + +func newMockClient() *mockClient { + return &mockClient{} +} diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 4a3c71df3bf..c9df4c1bab4 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -19,7 +19,6 @@ package elasticsearch import ( "context" - "encoding/base64" "errors" "fmt" "net/http" @@ -84,7 +83,7 @@ func NewClient( URL: s.URL, Username: s.Username, Password: s.Password, - APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), + APIKey: s.APIKey, Headers: s.Headers, TLS: s.TLS, Kerberos: s.Kerberos, diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 5c18a2b9a78..f0b5cbf04ae 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -37,13 +37,14 @@ output.elasticsearch: password: "{pwd}" ------------------------------------------------------------------------------ -To use an API key to connect to {es}, use `api_key`. +To use an API key to connect to {es}, use `api_key`. The value must be the ID of +the API key and the API key joined by a colon. ["source","yaml",subs="attributes,callouts"] ------------------------------------------------------------------------------ output.elasticsearch: hosts: ["https://localhost:9200"] - api_key: "KnR6yE41RrSowb0kQ0HWoA" + api_key: "VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw" ------------------------------------------------------------------------------ If the Elasticsearch nodes are defined by `IP:PORT`, then add `protocol: https` to the yaml file. @@ -135,8 +136,8 @@ The default value is 1. ===== `api_key` -Instead of using usernames and passwords, -you can use API keys to secure communication with {es}. +Instead of using usernames and passwords, you can use API keys to secure communication +with {es}. The value must be the ID of the API key and the API key joined by a colon. For more information, see <>. ===== `username`