From d15c665019cbe2e5a49451d7bcd32f191191e96b Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 7 May 2024 10:37:10 +0930 Subject: [PATCH 1/3] x-pack/filebeat/input/entityanalytics/provider/azuread: fix query handling --- CHANGELOG.next.asciidoc | 1 + .../provider/azuread/fetcher/graph/graph.go | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f293c504be5..92a79460b6a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -144,6 +144,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Restore netflow input on Windows {pull}39024[39024] - Upgrade azure-event-hubs-go and azure-storage-blob-go dependencies. {pull}38861[38861] - Fix concurrency/error handling bugs in the AWS S3 input that could drop data and prevent ingestion of large buckets. {pull}39131[39131] +- Fix EntraID query handling. {issue}39419[39419] {pull}39420[39420] *Heartbeat* diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go index 01d2d707025..efa2760cbd6 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go @@ -31,9 +31,10 @@ import ( const ( defaultAPIEndpoint = "https://graph.microsoft.com/v1.0" - defaultGroupsQuery = "$select=displayName,members" - defaultUsersQuery = "$select=accountEnabled,userPrincipalName,mail,displayName,givenName,surname,jobTitle,officeLocation,mobilePhone,businessPhones" - defaultDevicesQuery = "$select=accountEnabled,deviceId,displayName,operatingSystem,operatingSystemVersion,physicalIds,extensionAttributes,alternativeSecurityIds" + queryName = "$select" + defaultGroupsQuery = "displayName,members" + defaultUsersQuery = "accountEnabled,userPrincipalName,mail,displayName,givenName,surname,jobTitle,officeLocation,mobilePhone,businessPhones" + defaultDevicesQuery = "accountEnabled,deviceId,displayName,operatingSystem,operatingSystemVersion,physicalIds,extensionAttributes,alternativeSecurityIds" apiGroupType = "#microsoft.graph.group" apiUserType = "#microsoft.graph.user" @@ -353,21 +354,21 @@ func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) ( if err != nil { return nil, fmt.Errorf("invalid groups URL endpoint: %w", err) } - groupsURL.RawQuery = url.QueryEscape(formatQuery(c.Select.GroupQuery, defaultGroupsQuery)) + groupsURL.RawQuery = queryName + "=" + url.QueryEscape(formatQuery(c.Select.GroupQuery, defaultGroupsQuery)) f.groupsURL = groupsURL.String() usersURL, err := url.Parse(f.conf.APIEndpoint + "/users/delta") if err != nil { return nil, fmt.Errorf("invalid users URL endpoint: %w", err) } - usersURL.RawQuery = url.QueryEscape(formatQuery(c.Select.UserQuery, defaultUsersQuery)) + usersURL.RawQuery = queryName + "=" + url.QueryEscape(formatQuery(c.Select.UserQuery, defaultUsersQuery)) f.usersURL = usersURL.String() devicesURL, err := url.Parse(f.conf.APIEndpoint + "/devices/delta") if err != nil { return nil, fmt.Errorf("invalid devices URL endpoint: %w", err) } - devicesURL.RawQuery = url.QueryEscape(formatQuery(c.Select.DeviceQuery, defaultDevicesQuery)) + devicesURL.RawQuery = queryName + "=" + url.QueryEscape(formatQuery(c.Select.DeviceQuery, defaultDevicesQuery)) f.devicesURL = devicesURL.String() // The API takes a departure from the query approach here, so we @@ -386,7 +387,7 @@ func formatQuery(query []string, dflt string) string { if len(query) == 0 { return dflt } - return "$select=" + strings.Join(query, ",") + return strings.Join(query, ",") } // newUserFromAPI translates an API-representation of a user to a fetcher.User. From b01cce8bb463f409ba9d884b816b81b5d5b04a83 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 7 May 2024 10:52:58 +0930 Subject: [PATCH 2/3] fix mess --- x-pack/filebeat/input/awss3/input_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index c76e939424f..bf33073cb2a 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -8,8 +8,9 @@ import ( "errors" "testing" - aws "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/stretchr/testify/assert" + + aws "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) func TestGetProviderFromDomain(t *testing.T) { From 1a48bd2dd5fb19ecef293d134a7b9bf9dc011cd8 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 8 May 2024 07:23:04 +0930 Subject: [PATCH 3/3] address pr comment --- .../provider/azuread/fetcher/graph/graph.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go index efa2760cbd6..558e277d106 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go @@ -354,21 +354,21 @@ func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) ( if err != nil { return nil, fmt.Errorf("invalid groups URL endpoint: %w", err) } - groupsURL.RawQuery = queryName + "=" + url.QueryEscape(formatQuery(c.Select.GroupQuery, defaultGroupsQuery)) + groupsURL.RawQuery = formatQuery(queryName, c.Select.GroupQuery, defaultGroupsQuery) f.groupsURL = groupsURL.String() usersURL, err := url.Parse(f.conf.APIEndpoint + "/users/delta") if err != nil { return nil, fmt.Errorf("invalid users URL endpoint: %w", err) } - usersURL.RawQuery = queryName + "=" + url.QueryEscape(formatQuery(c.Select.UserQuery, defaultUsersQuery)) + usersURL.RawQuery = formatQuery(queryName, c.Select.UserQuery, defaultUsersQuery) f.usersURL = usersURL.String() devicesURL, err := url.Parse(f.conf.APIEndpoint + "/devices/delta") if err != nil { return nil, fmt.Errorf("invalid devices URL endpoint: %w", err) } - devicesURL.RawQuery = queryName + "=" + url.QueryEscape(formatQuery(c.Select.DeviceQuery, defaultDevicesQuery)) + devicesURL.RawQuery = formatQuery(queryName, c.Select.DeviceQuery, defaultDevicesQuery) f.devicesURL = devicesURL.String() // The API takes a departure from the query approach here, so we @@ -383,11 +383,12 @@ func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) ( return &f, nil } -func formatQuery(query []string, dflt string) string { - if len(query) == 0 { - return dflt +func formatQuery(name string, query []string, dflt string) string { + q := dflt + if len(query) != 0 { + q = strings.Join(query, ",") } - return strings.Join(query, ",") + return url.Values{name: []string{q}}.Encode() } // newUserFromAPI translates an API-representation of a user to a fetcher.User.