[Filebeat] Use new JsonBlob type in CredentialsJSON

CHANGELOG.next.asciidoc

@@ -261,6 +261,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
 - Fix a connection error in httpjson input. {pull}16123[16123]
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
+- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 
 *Filebeat*
 

libbeat/common/jsonblob.go

@@ -0,0 +1,46 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package common
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+// JSONBlob is a custom type that can unpack raw JSON strings or objects into
+// a json.RawMessage.
+type JSONBlob json.RawMessage
+
+func (b *JSONBlob) Unpack(v interface{}) error {
+	switch t := v.(type) {
+	case string:
+		*b = []byte(t)
+	default:
+		m, err := json.Marshal(v)
+		if err != nil {
+			return err
+		}
+		*b = m
+	}
+
+	if !json.Valid(*b) {
+		return errors.New("the field can't be converted to valid JSON")
+	}
+
+	return nil
+}

libbeat/common/jsonblob_test.go

@@ -0,0 +1,79 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigJSONBlob(t *testing.T) {
+	cases := []struct {
+		name        string
+		config      map[string]interface{}
+		expectedOut []byte
+		expectedErr string
+	}{
+		{
+			name: "successfully unpacks string",
+			config: map[string]interface{}{
+				"jsonBlob": `{"key":"value"}`,
+			},
+			expectedOut: []byte(`{"key":"value"}`),
+		},
+		{
+			name: "successfully unpacks map[string]interface{}",
+			config: map[string]interface{}{
+				"jsonBlob": map[string]interface{}{"key": "value"},
+			},
+			expectedOut: []byte(`{"key":"value"}`),
+		},
+		{
+			name: "successfully unpacks MapStr",
+			config: map[string]interface{}{
+				"jsonBlob": MapStr{"key": "value"},
+			},
+			expectedOut: []byte(`{"key":"value"}`),
+		},
+		{
+			name: "fails if can't be converted to json",
+			config: map[string]interface{}{
+				"jsonBlob": `invalid`,
+			},
+			expectedErr: "the field can't be converted to valid JSON accessing 'jsonBlob'",
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := MustNewConfigFrom(tc.config)
+			conf := struct {
+				JSONBlob JSONBlob `config:"jsonBlob"`
+			}{}
+			err := cfg.Unpack(&conf)
+			if tc.expectedErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, tc.expectedErr)
+			}
+			assert.EqualValues(t, string(tc.expectedOut), string(conf.JSONBlob))
+		})
+	}
+}

x-pack/filebeat/input/gcppubsub/config.go

@@ -10,6 +10,7 @@ import (
 	"os"
 
 	"github.com/elastic/beats/v7/filebeat/harvester"
+	"github.com/elastic/beats/v7/libbeat/common"
 
 	"cloud.google.com/go/pubsub"
 	"golang.org/x/oauth2/google"
@@ -35,7 +36,7 @@ type config struct {
 	CredentialsFile string `config:"credentials_file"`
 
 	// JSON blob containing authentication credentials and key.
-	CredentialsJSON []byte `config:"credentials_json"`
+	CredentialsJSON common.JSONBlob `config:"credentials_json"`
 
 	// Overrides the default Pub/Sub service address and disables TLS. For testing.
 	AlternativeHost string `config:"alternative_host"`

x-pack/filebeat/input/httpjson/config_oauth.go

@@ -18,6 +18,8 @@ import (
 	"golang.org/x/oauth2/clientcredentials"
 	"golang.org/x/oauth2/endpoints"
 	"golang.org/x/oauth2/google"
+
+	"github.com/elastic/beats/v7/libbeat/common"
 )
 
 // An oauth2Provider represents a supported oauth provider.
@@ -50,10 +52,10 @@ type oauth2Config struct {
 	TokenURL       string              `config:"token_url"`
 
 	// google specific
-	GoogleCredentialsFile  string `config:"google.credentials_file"`
-	GoogleCredentialsJSON  []byte `config:"google.credentials_json"`
-	GoogleJWTFile          string `config:"google.jwt_file"`
-	GoogleDelegatedAccount string `config:"google.delegated_account"`
+	GoogleCredentialsFile  string          `config:"google.credentials_file"`
+	GoogleCredentialsJSON  common.JSONBlob `config:"google.credentials_json"`
+	GoogleJWTFile          string          `config:"google.jwt_file"`
+	GoogleDelegatedAccount string          `config:"google.delegated_account"`
 
 	// microsoft azure specific
 	AzureTenantID string `config:"azure.tenant_id"`
@@ -163,9 +165,6 @@ func (o *oauth2Config) validateGoogleProvider() error {
 		if o.GoogleDelegatedAccount != "" {
 			return errors.New("invalid configuration: google.delegated_account can only be provided with a jwt_file")
 		}
-		if !json.Valid(o.GoogleCredentialsJSON) {
-			return errors.New("invalid configuration: google.credentials_json must be valid JSON")
-		}
 		return nil
 	}
 

x-pack/filebeat/input/httpjson/config_test.go

@@ -329,24 +329,24 @@ func TestConfigOauth2Validation(t *testing.T) {
 			input: map[string]interface{}{
 				"oauth2": map[string]interface{}{
 					"provider": "google",
-					"google.credentials_json": []byte(`{
+					"google.credentials_json": `{
 						"type":           "service_account",
 						"project_id":     "foo",
 						"private_key_id": "x",
 						"client_email":   "[email protected]",
 						"client_id":      "0"
-					}`),
+					}`,
 				},
 				"url": "localhost",
 			},
 		},
 		{
 			name:        "google must fail if credentials_json is not a valid JSON",
-			expectedErr: "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'",
+			expectedErr: "the field can't be converted to valid JSON accessing 'oauth2.google.credentials_json'",
 			input: map[string]interface{}{
 				"oauth2": map[string]interface{}{
 					"provider":                "google",
-					"google.credentials_json": []byte(`invalid`),
+					"google.credentials_json": `invalid`,
 				},
 				"url": "localhost",
 			},

x-pack/filebeat/input/httpjson/internal/v2/config_auth.go

@@ -18,6 +18,8 @@ import (
 	"golang.org/x/oauth2/clientcredentials"
 	"golang.org/x/oauth2/endpoints"
 	"golang.org/x/oauth2/google"
+
+	"github.com/elastic/beats/v7/libbeat/common"
 )
 
 type authConfig struct {
@@ -86,10 +88,10 @@ type oAuth2Config struct {
 	TokenURL       string              `config:"token_url"`
 
 	// google specific
-	GoogleCredentialsFile  string `config:"google.credentials_file"`
-	GoogleCredentialsJSON  []byte `config:"google.credentials_json"`
-	GoogleJWTFile          string `config:"google.jwt_file"`
-	GoogleDelegatedAccount string `config:"google.delegated_account"`
+	GoogleCredentialsFile  string          `config:"google.credentials_file"`
+	GoogleCredentialsJSON  common.JSONBlob `config:"google.credentials_json"`
+	GoogleJWTFile          string          `config:"google.jwt_file"`
+	GoogleDelegatedAccount string          `config:"google.delegated_account"`
 
 	// microsoft azure specific
 	AzureTenantID string `config:"azure.tenant_id"`
@@ -203,9 +205,6 @@ func (o *oAuth2Config) validateGoogleProvider() error {
 		if o.GoogleDelegatedAccount != "" {
 			return errors.New("google.delegated_account can only be provided with a jwt_file")
 		}
-		if !json.Valid(o.GoogleCredentialsJSON) {
-			return errors.New("google.credentials_json must be valid JSON")
-		}
 		return nil
 	}
 

x-pack/filebeat/input/httpjson/internal/v2/config_test.go

@@ -287,23 +287,23 @@ func TestConfigOauth2Validation(t *testing.T) {
 			input: map[string]interface{}{
 				"auth.oauth2": map[string]interface{}{
 					"provider": "google",
-					"google.credentials_json": []byte(`{
+					"google.credentials_json": `{
 						"type":           "service_account",
 						"project_id":     "foo",
 						"private_key_id": "x",
 						"client_email":   "[email protected]",
 						"client_id":      "0"
-					}`),
+					}`,
 				},
 			},
 		},
 		{
 			name:        "google must fail if credentials_json is not a valid JSON",
-			expectedErr: "google.credentials_json must be valid JSON accessing 'auth.oauth2'",
+			expectedErr: "the field can't be converted to valid JSON accessing 'auth.oauth2.google.credentials_json'",
 			input: map[string]interface{}{
 				"auth.oauth2": map[string]interface{}{
 					"provider":                "google",
-					"google.credentials_json": []byte(`invalid`),
+					"google.credentials_json": `invalid`,
 				},
 			},
 		},