From 5d239dba5fd4f1e8843ad347c7333777ecc1768e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 2 Jul 2024 15:54:27 +0930 Subject: [PATCH 1/2] x-pack/filebeat/input/gcs: allow absent credentials when using GCS with ADC --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/gcs/config.go | 36 ++++++++++++++- x-pack/filebeat/input/gcs/input_test.go | 44 +++++++++---------- .../input/gcs/testdata/gcs_creds.json | 0 4 files changed, 58 insertions(+), 23 deletions(-) create mode 100644 x-pack/filebeat/input/gcs/testdata/gcs_creds.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4787e8cecdb..fd7f83bcd9b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -298,6 +298,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add ability to remove request trace logs from entityanalytics input. {pull}40004[40004] - Relax constraint on Base DN in entity analytics Active Directory provider. {pull}40054[40054] - Enhance input state reporting for CEL evaluations that return a single error object in events. {pull}40083[40083] +- Allow absent credentials when using GCS with Application Default Credentials. {issue}39977[39977] {pull}40072[40072] *Auditbeat* diff --git a/x-pack/filebeat/input/gcs/config.go b/x-pack/filebeat/input/gcs/config.go index 846fe1fc94e..e362bce6aa4 100644 --- a/x-pack/filebeat/input/gcs/config.go +++ b/x-pack/filebeat/input/gcs/config.go @@ -5,8 +5,16 @@ package gcs import ( + "context" + "errors" + "fmt" + "io/fs" + "os" "time" + "cloud.google.com/go/pubsub" + "golang.org/x/oauth2/google" + "github.com/elastic/beats/v7/libbeat/common/match" ) @@ -17,7 +25,7 @@ type config struct { // ProjectId - Defines the project id of the concerned gcs bucket in Google Cloud. ProjectId string `config:"project_id" validate:"required"` // Auth - Defines the authentication mechanism to be used for accessing the gcs bucket. - Auth authConfig `config:"auth" validate:"required"` + Auth authConfig `config:"auth"` // MaxWorkers - Defines the maximum number of go routines that will be spawned. MaxWorkers *int `config:"max_workers,omitempty" validate:"max=5000"` // Poll - Defines if polling should be performed on the input bucket source. @@ -71,3 +79,29 @@ type fileCredentialsConfig struct { type jsonCredentialsConfig struct { AccountKey string `config:"account_key"` } + +func (c authConfig) Validate() error { + // credentials_file + if c.CredentialsFile != nil { + _, err := os.Stat(c.CredentialsFile.Path) + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("credentials_file is configured, but the file %q cannot be found", c.CredentialsFile.Path) + } else { + return nil + } + } + + // credentials_json + if c.CredentialsJSON != nil && len(c.CredentialsJSON.AccountKey) > 0 { + return nil + } + + // Application Default Credentials (ADC) + _, err := google.FindDefaultCredentials(context.Background(), pubsub.ScopePubSub) + if err == nil { + return nil + } + + return fmt.Errorf("no authentication credentials were configured or detected " + + "(credentials_file, credentials_json, and application default credentials (ADC))") +} diff --git a/x-pack/filebeat/input/gcs/input_test.go b/x-pack/filebeat/input/gcs/input_test.go index 41e64c031f9..64a548afd8c 100644 --- a/x-pack/filebeat/input/gcs/input_test.go +++ b/x-pack/filebeat/input/gcs/input_test.go @@ -49,7 +49,7 @@ func Test_StorageClient(t *testing.T) { name: "SingleBucketWithPoll_NoErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": true, "poll_interval": "5s", @@ -70,7 +70,7 @@ func Test_StorageClient(t *testing.T) { name: "SingleBucketWithoutPoll_NoErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": false, "poll_interval": "10s", @@ -91,7 +91,7 @@ func Test_StorageClient(t *testing.T) { name: "TwoBucketsWithPoll_NoErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": true, "poll_interval": "5s", @@ -117,7 +117,7 @@ func Test_StorageClient(t *testing.T) { name: "TwoBucketsWithoutPoll_NoErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": false, "poll_interval": "10s", @@ -143,7 +143,7 @@ func Test_StorageClient(t *testing.T) { name: "SingleBucketWithPoll_InvalidBucketErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": true, "poll_interval": "5s", @@ -161,7 +161,7 @@ func Test_StorageClient(t *testing.T) { name: "SingleBucketWithoutPoll_InvalidBucketErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": false, "poll_interval": "5s", @@ -179,7 +179,7 @@ func Test_StorageClient(t *testing.T) { name: "TwoBucketsWithPoll_InvalidBucketErr", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 2, "poll": true, "poll_interval": "5s", @@ -200,7 +200,7 @@ func Test_StorageClient(t *testing.T) { name: "SingleBucketWithPoll_InvalidConfigValue", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 5100, "poll": true, "poll_interval": "5s", @@ -218,7 +218,7 @@ func Test_StorageClient(t *testing.T) { name: "TwoBucketWithPoll_InvalidConfigValue", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 5100, "poll": true, "poll_interval": "5s", @@ -239,7 +239,7 @@ func Test_StorageClient(t *testing.T) { name: "SingleBucketWithPoll_parseJSON", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -261,7 +261,7 @@ func Test_StorageClient(t *testing.T) { name: "ReadJSON", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -282,7 +282,7 @@ func Test_StorageClient(t *testing.T) { name: "ReadOctetStreamJSON", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -302,7 +302,7 @@ func Test_StorageClient(t *testing.T) { name: "ReadNDJSON", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -322,7 +322,7 @@ func Test_StorageClient(t *testing.T) { name: "ReadMultilineGzJSON", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -342,7 +342,7 @@ func Test_StorageClient(t *testing.T) { name: "ReadJSONWithRootAsArray", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -364,7 +364,7 @@ func Test_StorageClient(t *testing.T) { name: "FilterByTimeStampEpoch", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -385,7 +385,7 @@ func Test_StorageClient(t *testing.T) { name: "FilterByFileSelectorRegexSingle", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -409,7 +409,7 @@ func Test_StorageClient(t *testing.T) { name: "FilterByFileSelectorRegexMulti", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -437,7 +437,7 @@ func Test_StorageClient(t *testing.T) { name: "ExpandEventListFromField", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -463,7 +463,7 @@ func Test_StorageClient(t *testing.T) { name: "MultiContainerWithMultiFileSelectors", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -496,7 +496,7 @@ func Test_StorageClient(t *testing.T) { name: "FilterByFileSelectorEmptyRegex", baseConfig: map[string]interface{}{ "project_id": "elastic-sa", - "auth.credentials_file.path": "/gcs_creds.json", + "auth.credentials_file.path": "testdata/gcs_creds.json", "max_workers": 1, "poll": true, "poll_interval": "5s", @@ -536,7 +536,7 @@ func Test_StorageClient(t *testing.T) { conf := config{} err := cfg.Unpack(&conf) if err != nil { - assert.EqualError(t, err, tt.isError.Error()) + assert.EqualError(t, err, fmt.Sprint(tt.isError)) return } input := newStatelessInput(conf) diff --git a/x-pack/filebeat/input/gcs/testdata/gcs_creds.json b/x-pack/filebeat/input/gcs/testdata/gcs_creds.json new file mode 100644 index 00000000000..e69de29bb2d From e1c6e460f4495fc6c98719473bdf201c52f5d2b6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 3 Jul 2024 12:40:05 +0930 Subject: [PATCH 2/2] address pr comment --- x-pack/filebeat/input/gcs/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/gcs/config.go b/x-pack/filebeat/input/gcs/config.go index e362bce6aa4..ed589e43df1 100644 --- a/x-pack/filebeat/input/gcs/config.go +++ b/x-pack/filebeat/input/gcs/config.go @@ -12,7 +12,7 @@ import ( "os" "time" - "cloud.google.com/go/pubsub" + "cloud.google.com/go/storage" "golang.org/x/oauth2/google" "github.com/elastic/beats/v7/libbeat/common/match" @@ -97,7 +97,7 @@ func (c authConfig) Validate() error { } // Application Default Credentials (ADC) - _, err := google.FindDefaultCredentials(context.Background(), pubsub.ScopePubSub) + _, err := google.FindDefaultCredentials(context.Background(), storage.ScopeReadOnly) if err == nil { return nil }