From e0ac1f2dffa2e4d91cd24591849fcd506e4ef0cd Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 21 Apr 2022 13:21:42 +0530 Subject: [PATCH 01/30] Azure AD Workload Identity Support - Azure Service Bus Scaler. Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 1 + .../v1alpha1/triggerauthentication_types.go | 13 +-- go.mod | 3 + go.sum | 7 ++ pkg/scalers/azure/azure_aad_auth.go | 30 ++++++ pkg/scalers/azure/azure_aad_podidentity.go | 11 --- .../azure/azure_aad_workload_identity.go | 97 +++++++++++++++++++ pkg/scalers/azure_servicebus_scaler.go | 40 +++++--- pkg/scalers/azure_servicebus_scaler_test.go | 13 +++ 9 files changed, 184 insertions(+), 31 deletions(-) create mode 100644 pkg/scalers/azure/azure_aad_auth.go create mode 100644 pkg/scalers/azure/azure_aad_workload_identity.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 523ce87e16e..cb153529255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md - **General:** Introduce new GCP Stackdriver Scaler ([#2661](https://github.com/kedacore/keda/issues/2661)) - **General:** Introduce new GCP Storage Scaler ([#2628](https://github.com/kedacore/keda/issues/2628)) - **General:** Provide support for authentication via Azure Key Vault ([#900](https://github.com/kedacore/keda/issues/900)|[#2733](https://github.com/kedacore/keda/issues/2733)) +- **General:** Support for Azure AD Workload Identity as a pod identity provider. ([2487](https://github.com/kedacore/keda/issues/2487)) - **General**: Support for `ValueMetricType` in `ScaledObject` for all scalers except CPU/Memory ([#2030](https://github.com/kedacore/keda/issues/2030)) ### Improvements diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index a04466f669d..26dcfe8f9ad 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -95,12 +95,13 @@ type PodIdentityProvider string // PodIdentityProviderNone specifies the default state when there is no Identity Provider // PodIdentityProvider specifies other available Identity providers const ( - PodIdentityProviderNone PodIdentityProvider = "none" - PodIdentityProviderAzure PodIdentityProvider = "azure" - PodIdentityProviderGCP PodIdentityProvider = "gcp" - PodIdentityProviderSpiffe PodIdentityProvider = "spiffe" - PodIdentityProviderAwsEKS PodIdentityProvider = "aws-eks" - PodIdentityProviderAwsKiam PodIdentityProvider = "aws-kiam" + PodIdentityProviderNone PodIdentityProvider = "none" + PodIdentityProviderAzure PodIdentityProvider = "azure" + PodIdentityProviderAzureWorkload PodIdentityProvider = "azure-workload" + PodIdentityProviderGCP PodIdentityProvider = "gcp" + PodIdentityProviderSpiffe PodIdentityProvider = "spiffe" + PodIdentityProviderAwsEKS PodIdentityProvider = "aws-eks" + PodIdentityProviderAwsKiam PodIdentityProvider = "aws-kiam" ) // PodIdentityAnnotationEKS specifies aws role arn for aws-eks Identity Provider diff --git a/go.mod b/go.mod index 74fbe4208fd..d43b7389f7b 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 + github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 github.com/DataDog/datadog-api-client-go v1.13.0 github.com/Huawei/gophercloud v1.0.21 github.com/Shopify/sarama v1.32.0 @@ -140,6 +141,7 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188 // indirect @@ -185,6 +187,7 @@ require ( github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.8 // indirect diff --git a/go.sum b/go.sum index 288b9e5890e..a1e6db32933 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c= +github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -420,6 +422,9 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= @@ -795,6 +800,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -850,6 +856,7 @@ github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/pkg/scalers/azure/azure_aad_auth.go b/pkg/scalers/azure/azure_aad_auth.go new file mode 100644 index 00000000000..cb1cacf52a5 --- /dev/null +++ b/pkg/scalers/azure/azure_aad_auth.go @@ -0,0 +1,30 @@ +/* +Copyright 2022 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +// AADToken is the token from Azure AD +type AADToken struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn string `json:"expires_in"` + ExpiresOn string `json:"expires_on"` + NotBefore string `json:"not_before"` + Resource string `json:"resource"` + TokenType string `json:"token_type"` + GrantedScopes []string `json:"grantedScopes"` + DeclinedScopes []string `json:"DeclinedScopes"` +} diff --git a/pkg/scalers/azure/azure_aad_podidentity.go b/pkg/scalers/azure/azure_aad_podidentity.go index 8a4f2566b8b..aa67d0e4899 100644 --- a/pkg/scalers/azure/azure_aad_podidentity.go +++ b/pkg/scalers/azure/azure_aad_podidentity.go @@ -47,14 +47,3 @@ func GetAzureADPodIdentityToken(ctx context.Context, httpClient util.HTTPDoer, a return token, nil } - -// AADToken is the token from Azure AD -type AADToken struct { - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - ExpiresIn string `json:"expires_in"` - ExpiresOn string `json:"expires_on"` - NotBefore string `json:"not_before"` - Resource string `json:"resource"` - TokenType string `json:"token_type"` -} diff --git a/pkg/scalers/azure/azure_aad_workload_identity.go b/pkg/scalers/azure/azure_aad_workload_identity.go new file mode 100644 index 00000000000..79874428a8e --- /dev/null +++ b/pkg/scalers/azure/azure_aad_workload_identity.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + + "github.com/AzureAD/microsoft-authentication-library-for-go/apps/confidential" +) + +// Azure AD Workload Identity Webhook will inject the following environment variables. +// * AZURE_CLIENT_ID - Client id set in the service account annotation +// * AZURE_TENANT_ID - Tenant id set in the service account annotation. If not defined, then tenant id provided via +// azure-wi-webhook-config will be used. +// * AZURE_FEDERATED_TOKEN_FILE - Service account token file path +// * AZURE_AUTHORITY_HOST - Azure Active Directory (AAD) endpoint. +const ( + azureClientIDEnv = "AZURE_CLIENT_ID" + azureTenantIDEnv = "AZURE_TENANT_ID" + azureFederatedTokenFileEnv = "AZURE_FEDERATED_TOKEN_FILE" + azureAuthrityHostEnv = "AZURE_AUTHORITY_HOST" +) + +// GetAzureADWorkloadIdentityToken returns the AADToken for resource +func GetAzureADWorkloadIdentityToken(ctx context.Context, resource string) (AADToken, error) { + clientID := os.Getenv(azureClientIDEnv) + tenantID := os.Getenv(azureTenantIDEnv) + tokenFilePath := os.Getenv(azureFederatedTokenFileEnv) + authorityHost := os.Getenv(azureAuthrityHostEnv) + + signedAssertion, err := readJWTFromFileSystem(tokenFilePath) + if err != nil { + return AADToken{}, fmt.Errorf("error reading service account token - %w", err) + } + + cred, err := confidential.NewCredFromAssertion(signedAssertion) + if err != nil { + return AADToken{}, fmt.Errorf("error getting credentials from service account token - %w", err) + } + + authorityOption := confidential.WithAuthority(fmt.Sprintf("%s%s/oauth2/token", authorityHost, tenantID)) + confidentialClient, err := confidential.New( + clientID, + cred, + authorityOption, + ) + if err != nil { + return AADToken{}, fmt.Errorf("error creating confidential client - %w", err) + } + + result, err := confidentialClient.AcquireTokenByCredential(ctx, []string{getScopedResource(resource)}) + if err != nil { + return AADToken{}, fmt.Errorf("error acquiring aad token - %w", err) + } + + return AADToken{ + AccessToken: result.AccessToken, + ExpiresOn: strconv.FormatInt(result.ExpiresOn.Unix(), 10), + GrantedScopes: result.GrantedScopes, + DeclinedScopes: result.DeclinedScopes, + }, nil +} + +func readJWTFromFileSystem(tokenFilePath string) (string, error) { + token, err := os.ReadFile(tokenFilePath) + if err != nil { + return "", err + } + return string(token), nil +} + +func getScopedResource(resource string) string { + resource = strings.TrimSuffix(resource, "/") + if !strings.HasSuffix(resource, ".default") { + resource += "/.default" + } + + return resource +} diff --git a/pkg/scalers/azure_servicebus_scaler.go b/pkg/scalers/azure_servicebus_scaler.go index 210bf1f96d8..de739cc27b2 100755 --- a/pkg/scalers/azure_servicebus_scaler.go +++ b/pkg/scalers/azure_servicebus_scaler.go @@ -45,6 +45,8 @@ const ( subscription entityType = 2 messageCountMetricName = "messageCount" defaultTargetMessageCount = 5 + // Service bus resource id is "https://servicebus.azure.net/" in all cloud environments + serviceBusResource = "https://servicebus.azure.net/" ) var azureServiceBusLog = logf.Log.WithName("azure_servicebus_scaler") @@ -155,7 +157,7 @@ func parseAzureServiceBusMetadata(config *ScalerConfig) (*azureServiceBusMetadat if len(meta.connection) == 0 { return nil, fmt.Errorf("no connection setting given") } - case kedav1alpha1.PodIdentityProviderAzure: + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: if val, ok := config.TriggerMetadata["namespace"]; ok { meta.namespace = val } else { @@ -224,24 +226,32 @@ func (s *azureServiceBusScaler) GetMetrics(ctx context.Context, metricName strin } type azureTokenProvider struct { - httpClient *http.Client - ctx context.Context + httpClient *http.Client + ctx context.Context + podIdentity kedav1alpha1.PodIdentityProvider } // GetToken implements TokenProvider interface for azureTokenProvider func (a azureTokenProvider) GetToken(uri string) (*auth.Token, error) { ctx := a.ctx - // Service bus resource id is "https://servicebus.azure.net/" in all cloud environments - token, err := azure.GetAzureADPodIdentityToken(ctx, a.httpClient, "https://servicebus.azure.net/") + + var token azure.AADToken + var err error + + switch a.podIdentity { + case kedav1alpha1.PodIdentityProviderAzure: + token, err = azure.GetAzureADPodIdentityToken(ctx, a.httpClient, serviceBusResource) + case kedav1alpha1.PodIdentityProviderAzureWorkload: + scopedResource := fmt.Sprintf("%s%s", serviceBusResource, ".default") + token, err = azure.GetAzureADWorkloadIdentityToken(ctx, scopedResource) + default: + err = fmt.Errorf("unknown pod identity provider") + } if err != nil { return nil, err } - return &auth.Token{ - TokenType: auth.CBSTokenTypeJWT, - Token: token.AccessToken, - Expiry: token.ExpiresOn, - }, nil + return auth.NewToken(auth.CBSTokenTypeJWT, token.AccessToken, token.ExpiresOn), nil } // Returns the length of the queue or subscription @@ -267,19 +277,21 @@ func (s *azureServiceBusScaler) getServiceBusNamespace(ctx context.Context) (*se var namespace *servicebus.Namespace var err error - if s.podIdentity == "" || s.podIdentity == kedav1alpha1.PodIdentityProviderNone { + switch s.podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: namespace, err = servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(s.metadata.connection)) if err != nil { return namespace, err } - } else if s.podIdentity == kedav1alpha1.PodIdentityProviderAzure { + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: namespace, err = servicebus.NewNamespace() if err != nil { return namespace, err } namespace.TokenProvider = azureTokenProvider{ - ctx: ctx, - httpClient: s.httpClient, + ctx: ctx, + httpClient: s.httpClient, + podIdentity: s.podIdentity, } namespace.Name = s.metadata.namespace } diff --git a/pkg/scalers/azure_servicebus_scaler_test.go b/pkg/scalers/azure_servicebus_scaler_test.go index dcf783c9272..9f7c877f394 100755 --- a/pkg/scalers/azure_servicebus_scaler_test.go +++ b/pkg/scalers/azure_servicebus_scaler_test.go @@ -97,6 +97,10 @@ var parseServiceBusMetadataDataset = []parseServiceBusMetadataTestData{ {map[string]string{"queueName": queueName}, true, queue, "", map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, // correct pod identity {map[string]string{"queueName": queueName, "namespace": namespaceName}, false, queue, defaultSuffix, map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, + // workload identity but missing namespace + {map[string]string{"queueName": queueName}, true, queue, "", map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // correct workload identity + {map[string]string{"queueName": queueName, "namespace": namespaceName}, false, queue, defaultSuffix, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, } var azServiceBusMetricIdentifiers = []azServiceBusMetricIdentifier{ @@ -133,6 +137,15 @@ var getServiceBusLengthTestScalers = []azureServiceBusScaler{ podIdentity: kedav1alpha1.PodIdentityProviderAzure, httpClient: commonHTTPClient, }, + { + metadata: &azureServiceBusMetadata{ + entityType: subscription, + topicName: topicName, + subscriptionName: subscriptionName, + }, + podIdentity: kedav1alpha1.PodIdentityProviderAzureWorkload, + httpClient: commonHTTPClient, + }, } func TestParseServiceBusMetadata(t *testing.T) { From 00aa60221d7555978cc6dfd401909d103eceaedc Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 21 Apr 2022 14:30:32 +0530 Subject: [PATCH 02/30] Azure AD Workload Identity Support - Azure Monitor Scaler. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure/azure_aad_auth.go | 5 ++++ .../azure/azure_aad_workload_identity.go | 21 ++++++++++++++++ pkg/scalers/azure/azure_monitor.go | 24 +++++++++---------- pkg/scalers/azure_monitor_scaler.go | 6 +++-- pkg/scalers/azure_monitor_scaler_test.go | 4 ++++ 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/pkg/scalers/azure/azure_aad_auth.go b/pkg/scalers/azure/azure_aad_auth.go index cb1cacf52a5..ed420ea500a 100644 --- a/pkg/scalers/azure/azure_aad_auth.go +++ b/pkg/scalers/azure/azure_aad_auth.go @@ -28,3 +28,8 @@ type AADToken struct { GrantedScopes []string `json:"grantedScopes"` DeclinedScopes []string `json:"DeclinedScopes"` } + +// OAuthToken implements the adal.OAuthTokenProvider interface. It returns the current access token. +func (aadToken AADToken) OAuthToken() string { + return aadToken.AccessToken +} diff --git a/pkg/scalers/azure/azure_aad_workload_identity.go b/pkg/scalers/azure/azure_aad_workload_identity.go index 79874428a8e..beb21a7f8a9 100644 --- a/pkg/scalers/azure/azure_aad_workload_identity.go +++ b/pkg/scalers/azure/azure_aad_workload_identity.go @@ -23,6 +23,8 @@ import ( "strconv" "strings" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/AzureAD/microsoft-authentication-library-for-go/apps/confidential" ) @@ -95,3 +97,22 @@ func getScopedResource(resource string) string { return resource } + +type ADWorkloadIdentityConfig struct { + ctx context.Context + resource string +} + +func NewAzureADWorkloadIdentityConfig(ctx context.Context, resource string) auth.AuthorizerConfig { + return &ADWorkloadIdentityConfig{ctx: ctx, resource: resource} +} + +// Authorizer implements the auth.AuthorizerConfig interface +func (aadWiConfig ADWorkloadIdentityConfig) Authorizer() (autorest.Authorizer, error) { + aadToken, err := GetAzureADWorkloadIdentityToken(aadWiConfig.ctx, aadWiConfig.resource) + if err != nil { + return nil, err + } + + return autorest.NewBearerAuthorizer(aadToken), nil +} diff --git a/pkg/scalers/azure/azure_monitor.go b/pkg/scalers/azure/azure_monitor.go index 2a9aebef653..c4fd65b9f75 100644 --- a/pkg/scalers/azure/azure_monitor.go +++ b/pkg/scalers/azure/azure_monitor.go @@ -68,13 +68,7 @@ var azureMonitorLog = logf.Log.WithName("azure_monitor_scaler") // GetAzureMetricValue returns the value of an Azure Monitor metric, rounded to the nearest int func GetAzureMetricValue(ctx context.Context, info MonitorInfo, podIdentity kedav1alpha1.PodIdentityProvider) (int64, error) { - var podIdentityEnabled = true - - if podIdentity == "" || podIdentity == kedav1alpha1.PodIdentityProviderNone { - podIdentityEnabled = false - } - - client := createMetricsClient(info, podIdentityEnabled) + client := createMetricsClient(ctx, info, podIdentity) requestPtr, err := createMetricsRequest(info) if err != nil { return -1, err @@ -83,21 +77,25 @@ func GetAzureMetricValue(ctx context.Context, info MonitorInfo, podIdentity keda return executeRequest(ctx, client, requestPtr) } -func createMetricsClient(info MonitorInfo, podIdentityEnabled bool) insights.MetricsClient { +func createMetricsClient(ctx context.Context, info MonitorInfo, podIdentity kedav1alpha1.PodIdentityProvider) insights.MetricsClient { client := insights.NewMetricsClientWithBaseURI(info.AzureResourceManagerEndpoint, info.SubscriptionID) var authConfig auth.AuthorizerConfig - if podIdentityEnabled { - config := auth.NewMSIConfig() + switch podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: + config := auth.NewClientCredentialsConfig(info.ClientID, info.ClientPassword, info.TenantID) config.Resource = info.AzureResourceManagerEndpoint + config.AADEndpoint = info.ActiveDirectoryEndpoint authConfig = config - } else { - config := auth.NewClientCredentialsConfig(info.ClientID, info.ClientPassword, info.TenantID) + case kedav1alpha1.PodIdentityProviderAzure: + config := auth.NewMSIConfig() config.Resource = info.AzureResourceManagerEndpoint - config.AADEndpoint = info.ActiveDirectoryEndpoint authConfig = config + case kedav1alpha1.PodIdentityProviderAzureWorkload: + authConfig = NewAzureADWorkloadIdentityConfig(ctx, info.AzureResourceManagerEndpoint) } + authorizer, _ := authConfig.Authorizer() client.Authorizer = authorizer diff --git a/pkg/scalers/azure_monitor_scaler.go b/pkg/scalers/azure_monitor_scaler.go index 21da3b06293..2666afc464a 100644 --- a/pkg/scalers/azure_monitor_scaler.go +++ b/pkg/scalers/azure_monitor_scaler.go @@ -176,7 +176,8 @@ func parseAzureMonitorMetadata(config *ScalerConfig) (*azureMonitorMetadata, err // parseAzurePodIdentityParams gets the activeDirectory clientID and password func parseAzurePodIdentityParams(config *ScalerConfig) (clientID string, clientPassword string, err error) { - if config.PodIdentity == "" || config.PodIdentity == kedav1alpha1.PodIdentityProviderNone { + switch config.PodIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: clientID, err = getParameterFromConfig(config, "activeDirectoryClientId", true) if err != nil || clientID == "" { return "", "", fmt.Errorf("no activeDirectoryClientId given") @@ -191,7 +192,8 @@ func parseAzurePodIdentityParams(config *ScalerConfig) (clientID string, clientP if len(clientPassword) == 0 { return "", "", fmt.Errorf("no activeDirectoryClientPassword given") } - } else if config.PodIdentity != kedav1alpha1.PodIdentityProviderAzure { + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: + default: return "", "", fmt.Errorf("azure Monitor doesn't support pod identity %s", config.PodIdentity) } diff --git a/pkg/scalers/azure_monitor_scaler_test.go b/pkg/scalers/azure_monitor_scaler_test.go index d787d1889fd..4109e563b2a 100644 --- a/pkg/scalers/azure_monitor_scaler_test.go +++ b/pkg/scalers/azure_monitor_scaler_test.go @@ -83,6 +83,10 @@ var testParseAzMonitorMetadata = []parseAzMonitorMetadataTestData{ {map[string]string{"resourceURI": "test/resource/uri", "tenantId": "123", "subscriptionId": "456", "resourceGroupName": "test", "metricName": "metric", "metricAggregationInterval": "0:15:0", "metricAggregationType": "Average", "targetValue": "5"}, false, map[string]string{}, map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, // wrong podIdentity {map[string]string{"resourceURI": "test/resource/uri", "tenantId": "123", "subscriptionId": "456", "resourceGroupName": "test", "metricName": "metric", "metricAggregationInterval": "0:15:0", "metricAggregationType": "Average", "targetValue": "5"}, true, map[string]string{}, map[string]string{}, kedav1alpha1.PodIdentityProvider("notAzure")}, + // connection with workload Identity + {map[string]string{"resourceURI": "test/resource/uri", "tenantId": "123", "subscriptionId": "456", "resourceGroupName": "test", "metricName": "metric", "metricAggregationInterval": "0:15:0", "metricAggregationType": "Average", "targetValue": "5"}, false, map[string]string{}, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // wrong workload Identity + {map[string]string{"resourceURI": "test/resource/uri", "tenantId": "123", "subscriptionId": "456", "resourceGroupName": "test", "metricName": "metric", "metricAggregationInterval": "0:15:0", "metricAggregationType": "Average", "targetValue": "5"}, true, map[string]string{}, map[string]string{}, kedav1alpha1.PodIdentityProvider("notAzureWorkload")}, // known azure cloud {map[string]string{"resourceURI": "test/resource/uri", "tenantId": "123", "subscriptionId": "456", "resourceGroupName": "test", "metricName": "metric", "metricAggregationInterval": "0:15:0", "metricAggregationType": "Average", "activeDirectoryClientId": "CLIENT_ID", "activeDirectoryClientPasswordFromEnv": "CLIENT_PASSWORD", "targetValue": "5", "metricNamespace": "namespace", "cloud": "azureChinaCloud"}, false, testAzMonitorResolvedEnv, map[string]string{}, ""}, // private cloud From 420129fd5dceb3b074e5e776c31542ffaa1032f0 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 21 Apr 2022 16:09:49 +0530 Subject: [PATCH 03/30] Azure AD Workload Identity Support - Azure Application Insights Scaler. Signed-off-by: Vighnesh Shenoy --- .../azure/azure_aad_workload_identity.go | 2 +- pkg/scalers/azure/azure_app_insights.go | 18 ++++++++----- pkg/scalers/azure/azure_app_insights_test.go | 27 ++++++++++++++----- pkg/scalers/azure_app_insights_scaler_test.go | 24 +++++++++++++++++ 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/pkg/scalers/azure/azure_aad_workload_identity.go b/pkg/scalers/azure/azure_aad_workload_identity.go index beb21a7f8a9..c860b8599e1 100644 --- a/pkg/scalers/azure/azure_aad_workload_identity.go +++ b/pkg/scalers/azure/azure_aad_workload_identity.go @@ -104,7 +104,7 @@ type ADWorkloadIdentityConfig struct { } func NewAzureADWorkloadIdentityConfig(ctx context.Context, resource string) auth.AuthorizerConfig { - return &ADWorkloadIdentityConfig{ctx: ctx, resource: resource} + return ADWorkloadIdentityConfig{ctx: ctx, resource: resource} } // Authorizer implements the auth.AuthorizerConfig interface diff --git a/pkg/scalers/azure/azure_app_insights.go b/pkg/scalers/azure/azure_app_insights.go index 5a8de634013..22a0371f97e 100644 --- a/pkg/scalers/azure/azure_app_insights.go +++ b/pkg/scalers/azure/azure_app_insights.go @@ -60,17 +60,21 @@ func toISO8601(time string) (string, error) { return fmt.Sprintf("PT%02dH%02dM", hours, minutes), nil } -func getAuthConfig(info AppInsightsInfo, podIdentity kedav1alpha1.PodIdentityProvider) auth.AuthorizerConfig { - if podIdentity == "" || podIdentity == kedav1alpha1.PodIdentityProviderNone { +func getAuthConfig(ctx context.Context, info AppInsightsInfo, podIdentity kedav1alpha1.PodIdentityProvider) auth.AuthorizerConfig { + switch podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: config := auth.NewClientCredentialsConfig(info.ClientID, info.ClientPassword, info.TenantID) config.Resource = info.AppInsightsResourceURL config.AADEndpoint = info.ActiveDirectoryEndpoint return config + case kedav1alpha1.PodIdentityProviderAzure: + config := auth.NewMSIConfig() + config.Resource = info.AppInsightsResourceURL + return config + case kedav1alpha1.PodIdentityProviderAzureWorkload: + return NewAzureADWorkloadIdentityConfig(ctx, info.AppInsightsResourceURL) } - - config := auth.NewMSIConfig() - config.Resource = info.AppInsightsResourceURL - return config + return nil } func extractAppInsightValue(info AppInsightsInfo, metric ApplicationInsightsMetric) (int64, error) { @@ -112,7 +116,7 @@ func queryParamsForAppInsightsRequest(info AppInsightsInfo) (map[string]interfac // GetAzureAppInsightsMetricValue returns the value of an Azure App Insights metric, rounded to the nearest int func GetAzureAppInsightsMetricValue(ctx context.Context, info AppInsightsInfo, podIdentity kedav1alpha1.PodIdentityProvider) (int64, error) { - config := getAuthConfig(info, podIdentity) + config := getAuthConfig(ctx, info, podIdentity) authorizer, err := config.Authorizer() if err != nil { return -1, err diff --git a/pkg/scalers/azure/azure_app_insights_test.go b/pkg/scalers/azure/azure_app_insights_test.go index 36b953d1ae1..b7006f13a7e 100644 --- a/pkg/scalers/azure/azure_app_insights_test.go +++ b/pkg/scalers/azure/azure_app_insights_test.go @@ -1,6 +1,7 @@ package azure import ( + "context" "testing" "github.com/Azure/go-autorest/autorest/azure/auth" @@ -72,28 +73,40 @@ func TestAzGetAzureAppInsightsMetricValue(t *testing.T) { type testAppInsightsAuthConfigTestData struct { testName string - expectMSI bool + config string info AppInsightsInfo podIdentity kedav1alpha1.PodIdentityProvider } +const ( + msiConfig = "msiConfig" + clientCredentialsConfig = "clientCredentialsConfig" + workloadIdentityConfig = "workloadIdentityConfig" +) + var testAppInsightsAuthConfigData = []testAppInsightsAuthConfigTestData{ - {"client credentials", false, AppInsightsInfo{ClientID: "1234", ClientPassword: "pw", TenantID: "5678"}, ""}, - {"client credentials - pod id none", false, AppInsightsInfo{ClientID: "1234", ClientPassword: "pw", TenantID: "5678"}, kedav1alpha1.PodIdentityProviderNone}, - {"azure pod identity", true, AppInsightsInfo{}, kedav1alpha1.PodIdentityProviderAzure}, + {"client credentials", clientCredentialsConfig, AppInsightsInfo{ClientID: "1234", ClientPassword: "pw", TenantID: "5678"}, ""}, + {"client credentials - pod id none", clientCredentialsConfig, AppInsightsInfo{ClientID: "1234", ClientPassword: "pw", TenantID: "5678"}, kedav1alpha1.PodIdentityProviderNone}, + {"azure pod identity", msiConfig, AppInsightsInfo{}, kedav1alpha1.PodIdentityProviderAzure}, + {"azure workload identity", workloadIdentityConfig, AppInsightsInfo{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, } func TestAzAppInfoGetAuthConfig(t *testing.T) { for _, testData := range testAppInsightsAuthConfigData { - authConfig := getAuthConfig(testData.info, testData.podIdentity) - if testData.expectMSI { + authConfig := getAuthConfig(context.TODO(), testData.info, testData.podIdentity) + switch testData.config { + case msiConfig: if _, ok := authConfig.(auth.MSIConfig); !ok { t.Errorf("Test %v; incorrect auth config. expected MSI config", testData.testName) } - } else { + case clientCredentialsConfig: if _, ok := authConfig.(auth.ClientCredentialsConfig); !ok { t.Errorf("Test: %v; incorrect auth config. expected client credentials config", testData.testName) } + case workloadIdentityConfig: + if _, ok := authConfig.(ADWorkloadIdentityConfig); !ok { + t.Errorf("Test: %v; incorrect auth config. expected ad workload identity config", testData.testName) + } } } } diff --git a/pkg/scalers/azure_app_insights_scaler_test.go b/pkg/scalers/azure_app_insights_scaler_test.go index 345b01b8b9c..6d6c63235e5 100644 --- a/pkg/scalers/azure_app_insights_scaler_test.go +++ b/pkg/scalers/azure_app_insights_scaler_test.go @@ -104,6 +104,30 @@ var azureAppInsightsScalerData = []azureAppInsightsScalerTestData{ "activeDirectoryClientId": "5678", "activeDirectoryClientPassword": "pw", }, }}, + {name: "correct pod identity", isError: false, config: ScalerConfig{ + TriggerMetadata: map[string]string{ + "targetValue": "11", "applicationInsightsId": "1234", "metricId": "unittest/test", "metricAggregationTimespan": "01:02", "metricAggregationType": "max", "metricFilter": "cloud/roleName eq 'test'", "tenantId": "1234", + }, + PodIdentity: kedav1alpha1.PodIdentityProviderAzure, + }}, + {name: "invalid pod Identity", isError: true, config: ScalerConfig{ + TriggerMetadata: map[string]string{ + "targetValue": "11", "applicationInsightsId": "1234", "metricId": "unittest/test", "metricAggregationTimespan": "01:02", "metricAggregationType": "max", "metricFilter": "cloud/roleName eq 'test'", "tenantId": "1234", + }, + PodIdentity: kedav1alpha1.PodIdentityProvider("notAzure"), + }}, + {name: "correct workload identity", isError: false, config: ScalerConfig{ + TriggerMetadata: map[string]string{ + "targetValue": "11", "applicationInsightsId": "1234", "metricId": "unittest/test", "metricAggregationTimespan": "01:02", "metricAggregationType": "max", "metricFilter": "cloud/roleName eq 'test'", "tenantId": "1234", + }, + PodIdentity: kedav1alpha1.PodIdentityProviderAzureWorkload, + }}, + {name: "invalid workload Identity", isError: true, config: ScalerConfig{ + TriggerMetadata: map[string]string{ + "targetValue": "11", "applicationInsightsId": "1234", "metricId": "unittest/test", "metricAggregationTimespan": "01:02", "metricAggregationType": "max", "metricFilter": "cloud/roleName eq 'test'", "tenantId": "1234", + }, + PodIdentity: kedav1alpha1.PodIdentityProvider("notAzureWorkload"), + }}, {name: "app insights id in auth", isError: false, config: ScalerConfig{ TriggerMetadata: map[string]string{ "targetValue": "11", "metricId": "unittest/test", "metricAggregationTimespan": "01:02", "metricAggregationType": "max", "metricFilter": "cloud/roleName eq 'test'", "tenantId": "1234", From 9575a3a88fa4426c567f6cb76a0a5a1a187f7fa4 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 21 Apr 2022 21:51:25 +0530 Subject: [PATCH 04/30] Azure AD Workload Identity Support - Azure Log Analytics Scaler. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure_log_analytics_scaler.go | 47 +++++++++++++++---- .../azure_log_analytics_scaler_test.go | 11 +++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/pkg/scalers/azure_log_analytics_scaler.go b/pkg/scalers/azure_log_analytics_scaler.go index 083f19bdc5b..e83d1963103 100644 --- a/pkg/scalers/azure_log_analytics_scaler.go +++ b/pkg/scalers/azure_log_analytics_scaler.go @@ -31,6 +31,7 @@ import ( "sync" "time" + "github.com/Azure/azure-amqp-common-go/v3/auth" v2beta2 "k8s.io/api/autoscaling/v2beta2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -79,13 +80,14 @@ type sessionCache struct { } type tokenData struct { - TokenType string `json:"token_type"` - ExpiresIn int `json:"expires_in,string"` - ExtExpiresIn int `json:"ext_expires_in,string"` - ExpiresOn int64 `json:"expires_on,string"` - NotBefore int64 `json:"not_before,string"` - Resource string `json:"resource"` - AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in,string"` + ExtExpiresIn int `json:"ext_expires_in,string"` + ExpiresOn int64 `json:"expires_on,string"` + NotBefore int64 `json:"not_before,string"` + Resource string `json:"resource"` + AccessToken string `json:"access_token"` + IsWorkloadIdentityToken bool `json:"isWorkloadIdentityToken"` } type metricsData struct { @@ -165,7 +167,7 @@ func parseAzureLogAnalyticsMetadata(config *ScalerConfig) (*azureLogAnalyticsMet meta.clientSecret = clientSecret meta.podIdentity = "" - case kedav1alpha1.PodIdentityProviderAzure: + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: meta.podIdentity = string(config.PodIdentity) default: return nil, fmt.Errorf("error parsing metadata. Details: Log Analytics Scaler doesn't support pod identity %s", config.PodIdentity) @@ -473,6 +475,10 @@ func (s *azureLogAnalyticsScaler) refreshAccessToken(ctx context.Context) (token return tokenData{}, err } + if tokenInfo.IsWorkloadIdentityToken { + return tokenInfo, nil + } + // Now, let's check we can use this token. If no, wait until we can use it currentTimeSec := time.Now().Unix() if currentTimeSec < tokenInfo.NotBefore { @@ -494,9 +500,30 @@ func (s *azureLogAnalyticsScaler) getAuthorizationToken(ctx context.Context) (to var err error var tokenInfo tokenData - if s.metadata.podIdentity == "" { + switch s.metadata.podIdentity { + case string(kedav1alpha1.PodIdentityProviderAzureWorkload): + aadToken, err := azure.GetAzureADWorkloadIdentityToken(ctx, s.metadata.logAnalyticsResourceURL) + if err != nil { + return tokenData{}, nil + } + + expiresOn, err := strconv.ParseInt(aadToken.ExpiresOn, 10, 64) + if err != nil { + return tokenData{}, nil + } + + tokenInfo = tokenData{ + TokenType: string(auth.CBSTokenTypeJWT), + AccessToken: aadToken.AccessToken, + ExpiresOn: expiresOn, + Resource: s.metadata.logAnalyticsResourceURL, + IsWorkloadIdentityToken: true, + } + + return tokenInfo, nil + case "", string(kedav1alpha1.PodIdentityProviderNone): body, statusCode, err = s.executeAADApicall(ctx) - } else { + case string(kedav1alpha1.PodIdentityProviderAzure): body, statusCode, err = s.executeIMDSApicall(ctx) } diff --git a/pkg/scalers/azure_log_analytics_scaler_test.go b/pkg/scalers/azure_log_analytics_scaler_test.go index 824af19fee7..29ae365523b 100644 --- a/pkg/scalers/azure_log_analytics_scaler_test.go +++ b/pkg/scalers/azure_log_analytics_scaler_test.go @@ -168,6 +168,17 @@ func TestLogAnalyticsParseMetadata(t *testing.T) { t.Error("Expected error but got success") } } + + // test with workload identity params should not fail + for _, testData := range testLogAnalyticsMetadataWithPodIdentity { + _, err := parseAzureLogAnalyticsMetadata(&ScalerConfig{ResolvedEnv: sampleLogAnalyticsResolvedEnv, TriggerMetadata: testData.metadata, AuthParams: LogAnalyticsAuthParams, PodIdentity: kedav1alpha1.PodIdentityProviderAzureWorkload}) + if err != nil && !testData.isError { + t.Error("Expected success but got error", err) + } + if testData.isError && err == nil { + t.Error("Expected error but got success") + } + } } func TestLogAnalyticsGetMetricSpecForScaling(t *testing.T) { From 66348de8e248c44d3b12f32348d8fc7fd67a8aee Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 02:05:33 +0530 Subject: [PATCH 05/30] Azure AD Workload Identity Support - Azure Data Explorer Scaler. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure/azure_aad_auth.go | 5 --- .../azure/azure_aad_workload_identity.go | 41 ++++++++++++++--- pkg/scalers/azure/azure_data_explorer.go | 44 ++++++++++++------- pkg/scalers/azure/azure_data_explorer_test.go | 20 +++++---- pkg/scalers/azure_data_explorer_scaler.go | 6 +-- .../azure_data_explorer_scaler_test.go | 17 +++++++ pkg/scaling/scale_handler.go | 2 +- 7 files changed, 97 insertions(+), 38 deletions(-) diff --git a/pkg/scalers/azure/azure_aad_auth.go b/pkg/scalers/azure/azure_aad_auth.go index ed420ea500a..cb1cacf52a5 100644 --- a/pkg/scalers/azure/azure_aad_auth.go +++ b/pkg/scalers/azure/azure_aad_auth.go @@ -28,8 +28,3 @@ type AADToken struct { GrantedScopes []string `json:"grantedScopes"` DeclinedScopes []string `json:"DeclinedScopes"` } - -// OAuthToken implements the adal.OAuthTokenProvider interface. It returns the current access token. -func (aadToken AADToken) OAuthToken() string { - return aadToken.AccessToken -} diff --git a/pkg/scalers/azure/azure_aad_workload_identity.go b/pkg/scalers/azure/azure_aad_workload_identity.go index c860b8599e1..fe3bb67a3cc 100644 --- a/pkg/scalers/azure/azure_aad_workload_identity.go +++ b/pkg/scalers/azure/azure_aad_workload_identity.go @@ -100,19 +100,50 @@ func getScopedResource(resource string) string { type ADWorkloadIdentityConfig struct { ctx context.Context - resource string + Resource string } func NewAzureADWorkloadIdentityConfig(ctx context.Context, resource string) auth.AuthorizerConfig { - return ADWorkloadIdentityConfig{ctx: ctx, resource: resource} + return ADWorkloadIdentityConfig{ctx: ctx, Resource: resource} } // Authorizer implements the auth.AuthorizerConfig interface func (aadWiConfig ADWorkloadIdentityConfig) Authorizer() (autorest.Authorizer, error) { - aadToken, err := GetAzureADWorkloadIdentityToken(aadWiConfig.ctx, aadWiConfig.resource) + return autorest.NewBearerAuthorizer(&ADWorkloadIdentityTokenProvider{ctx: aadWiConfig.ctx, Resource: aadWiConfig.Resource}), nil +} + +// ADWorkloadIdentityTokenProvider is a type that implements the adal.OAuthTokenProvider and adal.Refresher interfaces. +// The OAuthTokenProvider interface is used by the BearerAuthorizer to get the token when preparing the HTTP Header. +// The Refresher interface is used by the BearerAuthorizer to refresh the token. +type ADWorkloadIdentityTokenProvider struct { + ctx context.Context + Resource string + aadToken AADToken +} + +// OAuthToken is for implementing the adal.OAuthTokenProvider interface. It returns the current access token. +func (wiTokenProvider *ADWorkloadIdentityTokenProvider) OAuthToken() string { + return wiTokenProvider.aadToken.AccessToken +} + +// Refresh is for implementing the adal.Refresher interface +func (wiTokenProvider *ADWorkloadIdentityTokenProvider) Refresh() error { + aadToken, err := GetAzureADWorkloadIdentityToken(wiTokenProvider.ctx, wiTokenProvider.Resource) if err != nil { - return nil, err + return err } - return autorest.NewBearerAuthorizer(aadToken), nil + wiTokenProvider.aadToken = aadToken + return nil +} + +// RefreshExchange is for implementing the adal.Refresher interface +func (wiTokenProvider *ADWorkloadIdentityTokenProvider) RefreshExchange(resource string) error { + wiTokenProvider.Resource = resource + return wiTokenProvider.Refresh() +} + +// EnsureFresh is for implementing the adal.Refresher interface +func (wiTokenProvider *ADWorkloadIdentityTokenProvider) EnsureFresh() error { + return wiTokenProvider.Refresh() } diff --git a/pkg/scalers/azure/azure_data_explorer.go b/pkg/scalers/azure/azure_data_explorer.go index 9d1e0e8f1b2..7899bd7fbb0 100644 --- a/pkg/scalers/azure/azure_data_explorer.go +++ b/pkg/scalers/azure/azure_data_explorer.go @@ -27,6 +27,8 @@ import ( "github.com/Azure/azure-kusto-go/kusto/unsafe" "github.com/Azure/go-autorest/autorest/azure/auth" logf "sigs.k8s.io/controller-runtime/pkg/log" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) type DataExplorerMetadata struct { @@ -44,13 +46,18 @@ type DataExplorerMetadata struct { var azureDataExplorerLogger = logf.Log.WithName("azure_data_explorer_scaler") -func CreateAzureDataExplorerClient(metadata *DataExplorerMetadata) (*kusto.Client, error) { - authConfig, err := getDataExplorerAuthConfig(metadata) +func CreateAzureDataExplorerClient(ctx context.Context, metadata *DataExplorerMetadata) (*kusto.Client, error) { + authConfig, err := getDataExplorerAuthConfig(ctx, metadata) if err != nil { return nil, fmt.Errorf("failed to get data explorer auth config: %v", err) } - client, err := kusto.New(metadata.Endpoint, kusto.Authorization{Config: *authConfig}) + authorizer, err := authConfig.Authorizer() + if err != nil { + return nil, fmt.Errorf("failed to get authorizer: %v", err) + } + + client, err := kusto.New(metadata.Endpoint, kusto.Authorization{Authorizer: authorizer}) if err != nil { return nil, fmt.Errorf("failed to create kusto client: %v", err) } @@ -58,26 +65,31 @@ func CreateAzureDataExplorerClient(metadata *DataExplorerMetadata) (*kusto.Clien return client, nil } -func getDataExplorerAuthConfig(metadata *DataExplorerMetadata) (*auth.AuthorizerConfig, error) { +func getDataExplorerAuthConfig(ctx context.Context, metadata *DataExplorerMetadata) (auth.AuthorizerConfig, error) { var authConfig auth.AuthorizerConfig - if metadata.PodIdentity != "" { + switch metadata.PodIdentity { + case "", string(kedav1alpha1.PodIdentityProviderNone): + if metadata.ClientID != "" && metadata.ClientSecret != "" && metadata.TenantID != "" { + config := auth.NewClientCredentialsConfig(metadata.ClientID, metadata.ClientSecret, metadata.TenantID) + config.Resource = metadata.Endpoint + config.AADEndpoint = metadata.ActiveDirectoryEndpoint + azureDataExplorerLogger.V(1).Info("Creating Azure Data Explorer Client using clientID, clientSecret and tenantID") + + authConfig = config + return authConfig, nil + } + case string(kedav1alpha1.PodIdentityProviderAzure): config := auth.NewMSIConfig() config.Resource = metadata.Endpoint azureDataExplorerLogger.V(1).Info("Creating Azure Data Explorer Client using Pod Identity") authConfig = config - return &authConfig, nil - } - - if metadata.ClientID != "" && metadata.ClientSecret != "" && metadata.TenantID != "" { - config := auth.NewClientCredentialsConfig(metadata.ClientID, metadata.ClientSecret, metadata.TenantID) - config.Resource = metadata.Endpoint - config.AADEndpoint = metadata.ActiveDirectoryEndpoint - azureDataExplorerLogger.V(1).Info("Creating Azure Data Explorer Client using clientID, clientSecret and tenantID") - - authConfig = config - return &authConfig, nil + return authConfig, nil + case string(kedav1alpha1.PodIdentityProviderAzureWorkload): + azureDataExplorerLogger.V(1).Info("Creating Azure Data Explorer Client using Workload Identity") + authConfig = NewAzureADWorkloadIdentityConfig(ctx, metadata.Endpoint) + return authConfig, nil } return nil, fmt.Errorf("missing credentials. please reconfigure your scaled object metadata") diff --git a/pkg/scalers/azure/azure_data_explorer_test.go b/pkg/scalers/azure/azure_data_explorer_test.go index 8f467f6605c..6233e1bbd9a 100644 --- a/pkg/scalers/azure/azure_data_explorer_test.go +++ b/pkg/scalers/azure/azure_data_explorer_test.go @@ -17,6 +17,7 @@ limitations under the License. package azure import ( + "context" "testing" "github.com/Azure/azure-kusto-go/kusto/data/errors" @@ -36,13 +37,14 @@ type testGetDataExplorerAuthConfig struct { } var ( - clientID = "test_client_id" - rowName = "result" - rowType types.Column = "long" - rowValue int64 = 3 - podIdentity = "Azure" - secret = "test_secret" - tenantID = "test_tenant_id" + clientID = "test_client_id" + rowName = "result" + rowType types.Column = "long" + rowValue int64 = 3 + podIdentity = "azure" + workloadIdentity = "azure-workload" + secret = "test_secret" + tenantID = "test_tenant_id" ) var testExtractDataExplorerMetricValues = []testExtractDataExplorerMetricValue{ @@ -65,6 +67,8 @@ var testGetDataExplorerAuthConfigs = []testGetDataExplorerAuthConfig{ {testMetadata: &DataExplorerMetadata{ClientID: clientID, ClientSecret: secret, TenantID: tenantID}, isError: false}, // Auth with podIdentity - pass {testMetadata: &DataExplorerMetadata{PodIdentity: podIdentity}, isError: false}, + // Auth with podIdentity - pass + {testMetadata: &DataExplorerMetadata{PodIdentity: workloadIdentity}, isError: false}, // Empty metadata - fail {testMetadata: &DataExplorerMetadata{}, isError: true}, // Empty tenantID - fail @@ -89,7 +93,7 @@ func TestExtractDataExplorerMetricValue(t *testing.T) { func TestGetDataExplorerAuthConfig(t *testing.T) { for _, testData := range testGetDataExplorerAuthConfigs { - _, err := getDataExplorerAuthConfig(testData.testMetadata) + _, err := getDataExplorerAuthConfig(context.TODO(), testData.testMetadata) if err != nil && !testData.isError { t.Error("Expected success but got error", err) } diff --git a/pkg/scalers/azure_data_explorer_scaler.go b/pkg/scalers/azure_data_explorer_scaler.go index 940beebc364..08935ed68fc 100644 --- a/pkg/scalers/azure_data_explorer_scaler.go +++ b/pkg/scalers/azure_data_explorer_scaler.go @@ -46,7 +46,7 @@ const adxName = "azure-data-explorer" var dataExplorerLogger = logf.Log.WithName("azure_data_explorer_scaler") -func NewAzureDataExplorerScaler(config *ScalerConfig) (Scaler, error) { +func NewAzureDataExplorerScaler(ctx context.Context, config *ScalerConfig) (Scaler, error) { metricType, err := GetMetricTargetType(config) if err != nil { return nil, fmt.Errorf("error getting scaler metric type: %s", err) @@ -57,7 +57,7 @@ func NewAzureDataExplorerScaler(config *ScalerConfig) (Scaler, error) { return nil, fmt.Errorf("failed to parse azure data explorer metadata: %s", err) } - client, err := azure.CreateAzureDataExplorerClient(metadata) + client, err := azure.CreateAzureDataExplorerClient(ctx, metadata) if err != nil { return nil, fmt.Errorf("failed to create azure data explorer client: %s", err) } @@ -132,7 +132,7 @@ func parseAzureDataExplorerAuthParams(config *ScalerConfig) (*azure.DataExplorer metadata := azure.DataExplorerMetadata{} switch config.PodIdentity { - case kedav1alpha1.PodIdentityProviderAzure: + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: metadata.PodIdentity = string(config.PodIdentity) case "", kedav1alpha1.PodIdentityProviderNone: dataExplorerLogger.V(1).Info("Pod Identity is not provided. Trying to resolve clientId, clientSecret and tenantId.") diff --git a/pkg/scalers/azure_data_explorer_scaler_test.go b/pkg/scalers/azure_data_explorer_scaler_test.go index 432ce71561e..7c1879764c9 100644 --- a/pkg/scalers/azure_data_explorer_scaler_test.go +++ b/pkg/scalers/azure_data_explorer_scaler_test.go @@ -135,6 +135,23 @@ func TestDataExplorerParseMetadata(t *testing.T) { t.Error("Expected error but got success") } } + + // Auth through Workload Identity + for _, testData := range testDataExplorerMetadataWithPodIdentity { + _, err := parseAzureDataExplorerMetadata( + &ScalerConfig{ + ResolvedEnv: dataExplorerResolvedEnv, + TriggerMetadata: testData.metadata, + AuthParams: map[string]string{}, + PodIdentity: kedav1alpha1.PodIdentityProviderAzureWorkload}) + + if err != nil && !testData.isError { + t.Error("Expected success but got error", err) + } + if testData.isError && err == nil { + t.Error("Expected error but got success") + } + } } func TestDataExplorerGetMetricSpecForScaling(t *testing.T) { diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go index faba46a1449..c8865809523 100644 --- a/pkg/scaling/scale_handler.go +++ b/pkg/scaling/scale_handler.go @@ -366,7 +366,7 @@ func buildScaler(ctx context.Context, client client.Client, triggerType string, case "azure-blob": return scalers.NewAzureBlobScaler(config) case "azure-data-explorer": - return scalers.NewAzureDataExplorerScaler(config) + return scalers.NewAzureDataExplorerScaler(ctx, config) case "azure-eventhub": return scalers.NewAzureEventHubScaler(config) case "azure-log-analytics": From e8cece0bf0b933e692c329a3921ade1a1840161b Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 12:57:01 +0530 Subject: [PATCH 06/30] Azure AD Workload Identity Support - Azure Event Hub Scaler. Signed-off-by: Vighnesh Shenoy --- .../azure/azure_aad_workload_identity.go | 16 ++++++++ pkg/scalers/azure/azure_eventhub.go | 40 ++++++++++++------- pkg/scalers/azure_eventhub_scaler.go | 21 +++++++--- pkg/scalers/azure_eventhub_scaler_test.go | 13 +++++- pkg/scaling/scale_handler.go | 2 +- 5 files changed, 69 insertions(+), 23 deletions(-) diff --git a/pkg/scalers/azure/azure_aad_workload_identity.go b/pkg/scalers/azure/azure_aad_workload_identity.go index fe3bb67a3cc..92dcfab210f 100644 --- a/pkg/scalers/azure/azure_aad_workload_identity.go +++ b/pkg/scalers/azure/azure_aad_workload_identity.go @@ -23,6 +23,7 @@ import ( "strconv" "strings" + amqpAuth "github.com/Azure/azure-amqp-common-go/v3/auth" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/AzureAD/microsoft-authentication-library-for-go/apps/confidential" @@ -121,6 +122,10 @@ type ADWorkloadIdentityTokenProvider struct { aadToken AADToken } +func NewADWorkloadIdentityTokenProvider(ctx context.Context, resource string) *ADWorkloadIdentityTokenProvider { + return &ADWorkloadIdentityTokenProvider{ctx: ctx, Resource: resource} +} + // OAuthToken is for implementing the adal.OAuthTokenProvider interface. It returns the current access token. func (wiTokenProvider *ADWorkloadIdentityTokenProvider) OAuthToken() string { return wiTokenProvider.aadToken.AccessToken @@ -147,3 +152,14 @@ func (wiTokenProvider *ADWorkloadIdentityTokenProvider) RefreshExchange(resource func (wiTokenProvider *ADWorkloadIdentityTokenProvider) EnsureFresh() error { return wiTokenProvider.Refresh() } + +// GetToken is for implementing the auth.TokenProvider interface +func (wiTokenProvider *ADWorkloadIdentityTokenProvider) GetToken(uri string) (*amqpAuth.Token, error) { + err := wiTokenProvider.Refresh() + if err != nil { + return nil, err + } + + return amqpAuth.NewToken(amqpAuth.CBSTokenTypeJWT, wiTokenProvider.aadToken.AccessToken, + wiTokenProvider.aadToken.ExpiresOn), nil +} diff --git a/pkg/scalers/azure/azure_eventhub.go b/pkg/scalers/azure/azure_eventhub.go index 624ab47075d..67f671e6c26 100644 --- a/pkg/scalers/azure/azure_eventhub.go +++ b/pkg/scalers/azure/azure_eventhub.go @@ -1,6 +1,7 @@ package azure import ( + "context" "errors" "fmt" "strings" @@ -8,6 +9,8 @@ import ( "github.com/Azure/azure-amqp-common-go/v3/aad" eventhub "github.com/Azure/azure-event-hubs-go/v3" "github.com/Azure/go-autorest/autorest/azure" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) // EventHubInfo to keep event hub connection and resources @@ -19,15 +22,20 @@ type EventHubInfo struct { Namespace string EventHubName string CheckpointStrategy string - Cloud string ServiceBusEndpointSuffix string ActiveDirectoryEndpoint string + EventHubResourceURL string + PodIdentity kedav1alpha1.PodIdentityProvider } +const ( + DefaultEventhubResourceURL = "https://eventhubs.azure.net/" +) + // GetEventHubClient returns eventhub client -func GetEventHubClient(info EventHubInfo) (*eventhub.Hub, error) { +func GetEventHubClient(ctx context.Context, info EventHubInfo) (*eventhub.Hub, error) { // The user wants to use a connectionstring, not a pod identity - if info.EventHubConnection != "" { + if info.PodIdentity == "" || info.PodIdentity == kedav1alpha1.PodIdentityProviderNone { hub, err := eventhub.NewHubFromConnectionString(info.EventHubConnection) if err != nil { return nil, fmt.Errorf("failed to create hub client: %s", err) @@ -36,23 +44,25 @@ func GetEventHubClient(info EventHubInfo) (*eventhub.Hub, error) { } env := azure.Environment{ActiveDirectoryEndpoint: info.ActiveDirectoryEndpoint, ServiceBusEndpointSuffix: info.ServiceBusEndpointSuffix} + hubEnvOptions := eventhub.HubWithEnvironment(env) + if info.PodIdentity == kedav1alpha1.PodIdentityProviderAzure { + // Since there is no connectionstring, then user wants to use pod identity + // Internally, the JWTProvider will use Managed Service Identity to authenticate if no Service Principal info supplied + envJWTProviderOption := aad.JWTProviderWithAzureEnvironment(&env) + resourceURLJWTProviderOption := aad.JWTProviderWithResourceURI(info.EventHubResourceURL) + provider, aadErr := aad.NewJWTProvider(envJWTProviderOption, resourceURLJWTProviderOption) - // Since there is no connectionstring, then user wants to use pod identity - // Internally, the JWTProvider will use Managed Service Identity to authenticate if no Service Principal info supplied - provider, aadErr := aad.NewJWTProvider(func(config *aad.TokenProviderConfiguration) error { - if config.Env == nil { - config.Env = &env + if aadErr == nil { + return eventhub.NewHub(info.Namespace, info.EventHubName, provider, hubEnvOptions) } - return nil - }) - - hubEnvOptions := eventhub.HubWithEnvironment(env) - if aadErr == nil { - return eventhub.NewHub(info.Namespace, info.EventHubName, provider, hubEnvOptions) + return nil, aadErr } - return nil, aadErr + // Workload Identity case + provider := NewADWorkloadIdentityTokenProvider(ctx, info.EventHubResourceURL) + + return eventhub.NewHub(info.Namespace, info.EventHubName, provider, hubEnvOptions) } // ParseAzureEventHubConnectionString parses Event Hub connection string into (namespace, name) diff --git a/pkg/scalers/azure_eventhub_scaler.go b/pkg/scalers/azure_eventhub_scaler.go index a3f2875d147..a94717f8042 100644 --- a/pkg/scalers/azure_eventhub_scaler.go +++ b/pkg/scalers/azure_eventhub_scaler.go @@ -23,6 +23,7 @@ import ( "math" "net/http" "strconv" + "strings" eventhub "github.com/Azure/azure-event-hubs-go/v3" "github.com/Azure/azure-storage-blob-go/azblob" @@ -64,7 +65,7 @@ type eventHubMetadata struct { } // NewAzureEventHubScaler creates a new scaler for eventHub -func NewAzureEventHubScaler(config *ScalerConfig) (Scaler, error) { +func NewAzureEventHubScaler(ctx context.Context, config *ScalerConfig) (Scaler, error) { metricType, err := GetMetricTargetType(config) if err != nil { return nil, fmt.Errorf("error getting scaler metric type: %s", err) @@ -75,7 +76,7 @@ func NewAzureEventHubScaler(config *ScalerConfig) (Scaler, error) { return nil, fmt.Errorf("unable to get eventhub metadata: %s", err) } - hub, err := azure.GetEventHubClient(parsedMetadata.eventHubInfo) + hub, err := azure.GetEventHubClient(ctx, parsedMetadata.eventHubInfo) if err != nil { return nil, fmt.Errorf("unable to get eventhub client: %s", err) } @@ -129,9 +130,15 @@ func parseAzureEventHubMetadata(config *ScalerConfig) (*eventHubMetadata, error) meta.eventHubInfo.BlobContainer = val } - meta.eventHubInfo.Cloud = azure.DefaultCloud + meta.eventHubInfo.EventHubResourceURL = azure.DefaultEventhubResourceURL if val, ok := config.TriggerMetadata["cloud"]; ok { - meta.eventHubInfo.Cloud = val + if strings.EqualFold(val, azure.PrivateCloud) { + if resourceURL, ok := config.TriggerMetadata["eventHubResourceURL"]; ok { + meta.eventHubInfo.EventHubResourceURL = resourceURL + } else { + return nil, fmt.Errorf("eventHubResourceURL must be provided for %s cloud type", azure.PrivateCloud) + } + } } serviceBusEndpointSuffixProvider := func(env az.Environment) (string, error) { @@ -149,7 +156,9 @@ func parseAzureEventHubMetadata(config *ScalerConfig) (*eventHubMetadata, error) } meta.eventHubInfo.ActiveDirectoryEndpoint = activeDirectoryEndpoint - if config.PodIdentity == "" || config.PodIdentity == v1alpha1.PodIdentityProviderNone { + meta.eventHubInfo.PodIdentity = config.PodIdentity + switch config.PodIdentity { + case "", v1alpha1.PodIdentityProviderNone: if config.AuthParams["connection"] != "" { meta.eventHubInfo.EventHubConnection = config.AuthParams["connection"] } else if config.TriggerMetadata["connectionFromEnv"] != "" { @@ -159,7 +168,7 @@ func parseAzureEventHubMetadata(config *ScalerConfig) (*eventHubMetadata, error) if len(meta.eventHubInfo.EventHubConnection) == 0 { return nil, fmt.Errorf("no event hub connection string given") } - } else { + case v1alpha1.PodIdentityProviderAzure, v1alpha1.PodIdentityProviderAzureWorkload: if config.TriggerMetadata["eventHubNamespace"] != "" { meta.eventHubInfo.Namespace = config.TriggerMetadata["eventHubNamespace"] } else if config.TriggerMetadata["eventHubNamespaceFromEnv"] != "" { diff --git a/pkg/scalers/azure_eventhub_scaler_test.go b/pkg/scalers/azure_eventhub_scaler_test.go index 7e2eda66f87..e0c302130c0 100644 --- a/pkg/scalers/azure_eventhub_scaler_test.go +++ b/pkg/scalers/azure_eventhub_scaler_test.go @@ -110,7 +110,18 @@ func TestParseEventHubMetadata(t *testing.T) { } for _, testData := range parseEventHubMetadataDatasetWithPodIdentity { - _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: "Azure"}) + _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: "azure"}) + + if err != nil && !testData.isError { + t.Errorf("Expected success but got error: %s", err) + } + if testData.isError && err == nil { + t.Error("Expected error and got success") + } + } + + for _, testData := range parseEventHubMetadataDatasetWithPodIdentity { + _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: "azure-workload"}) if err != nil && !testData.isError { t.Errorf("Expected success but got error: %s", err) diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go index c8865809523..3d5c39ff8ad 100644 --- a/pkg/scaling/scale_handler.go +++ b/pkg/scaling/scale_handler.go @@ -368,7 +368,7 @@ func buildScaler(ctx context.Context, client client.Client, triggerType string, case "azure-data-explorer": return scalers.NewAzureDataExplorerScaler(ctx, config) case "azure-eventhub": - return scalers.NewAzureEventHubScaler(config) + return scalers.NewAzureEventHubScaler(ctx, config) case "azure-log-analytics": return scalers.NewAzureLogAnalyticsScaler(config) case "azure-monitor": From 35611d1a6a497ba9a8d6299e2a9c13ecaeaa2b1e Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 13:51:40 +0530 Subject: [PATCH 07/30] Azure AD Pod Identity & Workload Identity Support - Azure Key Vault. Signed-off-by: Vighnesh Shenoy --- .../v1alpha1/triggerauthentication_types.go | 5 +- apis/keda/v1alpha1/zz_generated.deepcopy.go | 10 ++-- ...keda.sh_clustertriggerauthentications.yaml | 1 - .../bases/keda.sh_triggerauthentications.yaml | 1 - .../resolver/azure_keyvault_handler.go | 52 ++++++++++++++----- .../resolver/azure_keyvault_handler_test.go | 2 +- pkg/scaling/resolver/scale_resolvers.go | 2 +- 7 files changed, 48 insertions(+), 25 deletions(-) diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index 26dcfe8f9ad..24f4ea44c02 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -181,9 +181,10 @@ type VaultSecret struct { // AzureKeyVault is used to authenticate using Azure Key Vault type AzureKeyVault struct { - VaultURI string `json:"vaultUri"` + VaultURI string `json:"vaultUri"` + Secrets []AzureKeyVaultSecret `json:"secrets"` + // +optional Credentials *AzureKeyVaultCredentials `json:"credentials"` - Secrets []AzureKeyVaultSecret `json:"secrets"` // +optional Cloud *AzureKeyVaultCloudInfo `json:"cloud"` } diff --git a/apis/keda/v1alpha1/zz_generated.deepcopy.go b/apis/keda/v1alpha1/zz_generated.deepcopy.go index fc63e23d1f9..a7d7732de17 100644 --- a/apis/keda/v1alpha1/zz_generated.deepcopy.go +++ b/apis/keda/v1alpha1/zz_generated.deepcopy.go @@ -95,16 +95,16 @@ func (in *AuthSecretTargetRef) DeepCopy() *AuthSecretTargetRef { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureKeyVault) DeepCopyInto(out *AzureKeyVault) { *out = *in - if in.Credentials != nil { - in, out := &in.Credentials, &out.Credentials - *out = new(AzureKeyVaultCredentials) - (*in).DeepCopyInto(*out) - } if in.Secrets != nil { in, out := &in.Secrets, &out.Secrets *out = make([]AzureKeyVaultSecret, len(*in)) copy(*out, *in) } + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(AzureKeyVaultCredentials) + (*in).DeepCopyInto(*out) + } if in.Cloud != nil { in, out := &in.Cloud, &out.Cloud *out = new(AzureKeyVaultCloudInfo) diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index a07a808c67b..0aed26b43a7 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -116,7 +116,6 @@ spec: vaultUri: type: string required: - - credentials - secrets - vaultUri type: object diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index a8ad47f2279..f54cbd72e69 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -115,7 +115,6 @@ spec: vaultUri: type: string required: - - credentials - secrets - vaultUri type: object diff --git a/pkg/scaling/resolver/azure_keyvault_handler.go b/pkg/scaling/resolver/azure_keyvault_handler.go index 68784728091..fb82b626c4d 100644 --- a/pkg/scaling/resolver/azure_keyvault_handler.go +++ b/pkg/scaling/resolver/azure_keyvault_handler.go @@ -34,32 +34,28 @@ import ( type AzureKeyVaultHandler struct { vault *kedav1alpha1.AzureKeyVault keyvaultClient *keyvault.BaseClient + podIdentity kedav1alpha1.PodIdentityProvider } -func NewAzureKeyVaultHandler(v *kedav1alpha1.AzureKeyVault) *AzureKeyVaultHandler { +func NewAzureKeyVaultHandler(v *kedav1alpha1.AzureKeyVault, podIdentity kedav1alpha1.PodIdentityProvider) *AzureKeyVaultHandler { return &AzureKeyVaultHandler{ - vault: v, + vault: v, + podIdentity: podIdentity, } } func (vh *AzureKeyVaultHandler) Initialize(ctx context.Context, client client.Client, logger logr.Logger, triggerNamespace string) error { - clientID := vh.vault.Credentials.ClientID - tenantID := vh.vault.Credentials.TenantID - - clientSecretName := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Name - clientSecretKey := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Key - clientSecret := resolveAuthSecret(ctx, client, logger, clientSecretName, triggerNamespace, clientSecretKey) - - clientCredentialsConfig := auth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) - keyvaultResourceURL, activeDirectoryEndpoint, err := vh.getPropertiesForCloud() if err != nil { return err } - clientCredentialsConfig.Resource = keyvaultResourceURL - clientCredentialsConfig.AADEndpoint = activeDirectoryEndpoint - authorizer, err := clientCredentialsConfig.Authorizer() + authConfig := vh.getAuthConfig(ctx, client, logger, triggerNamespace, keyvaultResourceURL, activeDirectoryEndpoint) + if err != nil { + return err + } + + authorizer, err := authConfig.Authorizer() if err != nil { return err } @@ -105,3 +101,31 @@ func (vh *AzureKeyVaultHandler) getPropertiesForCloud() (string, string, error) return env.ResourceIdentifiers.KeyVault, env.ActiveDirectoryEndpoint, nil } + +func (vh *AzureKeyVaultHandler) getAuthConfig(ctx context.Context, client client.Client, logger logr.Logger, + triggerNamespace, keyVaultResourceURL, activeDirectoryEndpoint string) auth.AuthorizerConfig { + switch vh.podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: + clientID := vh.vault.Credentials.ClientID + tenantID := vh.vault.Credentials.TenantID + + clientSecretName := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Name + clientSecretKey := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Key + clientSecret := resolveAuthSecret(ctx, client, logger, clientSecretName, triggerNamespace, clientSecretKey) + + config := auth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) + config.Resource = keyVaultResourceURL + config.AADEndpoint = activeDirectoryEndpoint + + return config + case kedav1alpha1.PodIdentityProviderAzure: + config := auth.NewMSIConfig() + config.Resource = keyVaultResourceURL + + return config + case kedav1alpha1.PodIdentityProviderAzureWorkload: + return azure.NewAzureADWorkloadIdentityConfig(ctx, keyVaultResourceURL) + } + + return nil +} diff --git a/pkg/scaling/resolver/azure_keyvault_handler_test.go b/pkg/scaling/resolver/azure_keyvault_handler_test.go index 4a6740f567d..2f0e1e1bc6a 100644 --- a/pkg/scaling/resolver/azure_keyvault_handler_test.go +++ b/pkg/scaling/resolver/azure_keyvault_handler_test.go @@ -110,7 +110,7 @@ var testDataset = []testData{ func TestGetPropertiesForCloud(t *testing.T) { for _, testData := range testDataset { - vh := NewAzureKeyVaultHandler(&testData.vault) + vh := NewAzureKeyVaultHandler(&testData.vault, kedav1alpha1.PodIdentityProviderNone) kvResourceURL, adEndpoint, err := vh.getPropertiesForCloud() diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index a0015a117a1..d93a7c24280 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -211,7 +211,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } } if triggerAuthSpec.AzureKeyVault != nil && len(triggerAuthSpec.AzureKeyVault.Secrets) > 0 { - vaultHandler := NewAzureKeyVaultHandler(triggerAuthSpec.AzureKeyVault) + vaultHandler := NewAzureKeyVaultHandler(triggerAuthSpec.AzureKeyVault, podIdentity) err := vaultHandler.Initialize(ctx, client, logger, triggerNamespace) if err != nil { logger.Error(err, "Error authenticating to Azure Key Vault", "triggerAuthRef.Name", triggerAuthRef.Name) From eca9ac6bca57ea399a36ee1df8f1e2651c0723d4 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 14:52:21 +0530 Subject: [PATCH 08/30] Fix event hub unit tests. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure_eventhub_scaler_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/scalers/azure_eventhub_scaler_test.go b/pkg/scalers/azure_eventhub_scaler_test.go index e0c302130c0..33d3b54c067 100644 --- a/pkg/scalers/azure_eventhub_scaler_test.go +++ b/pkg/scalers/azure_eventhub_scaler_test.go @@ -20,6 +20,7 @@ const ( storageConnectionSetting = "testStorageConnectionSetting" serviceBusEndpointSuffix = "serviceBusEndpointSuffix" activeDirectoryEndpoint = "activeDirectoryEndpoint" + eventHubResourceURL = "eventHubResourceURL" testEventHubNamespace = "kedatesteventhub" testEventHubName = "eventhub1" checkpointFormat = "{\"SequenceNumber\":%d,\"PartitionId\":\"%s\"}" @@ -68,18 +69,21 @@ var parseEventHubMetadataDatasetWithPodIdentity = []parseEventHubMetadataTestDat // metadata with cloud specified {map[string]string{"storageConnectionFromEnv": storageConnectionSetting, "consumerGroup": eventHubConsumerGroup, "unprocessedEventThreshold": "15", "eventHubName": testEventHubName, "eventHubNamespace": testEventHubNamespace, "cloud": "azurePublicCloud"}, false}, - // metadata with private cloud missing service bus endpoint suffix and active directory endpoint + // metadata with private cloud missing service bus endpoint suffix and active directory endpoint and eventHubResourceURL {map[string]string{"storageConnectionFromEnv": storageConnectionSetting, "consumerGroup": eventHubConsumerGroup, "unprocessedEventThreshold": "15", "eventHubName": testEventHubName, "eventHubNamespace": testEventHubNamespace, "cloud": "private"}, true}, - // metadata with private cloud missing active directory endpoint + // metadata with private cloud missing active directory endpoint and resourceURL {map[string]string{"storageConnectionFromEnv": storageConnectionSetting, "consumerGroup": eventHubConsumerGroup, "unprocessedEventThreshold": "15", "eventHubName": testEventHubName, "eventHubNamespace": testEventHubNamespace, "cloud": "private", "endpointSuffix": serviceBusEndpointSuffix}, true}, - // metadata with private cloud missing service bus endpoint suffix + // metadata with private cloud missing service bus endpoint suffix and resource URL {map[string]string{"storageConnectionFromEnv": storageConnectionSetting, "consumerGroup": eventHubConsumerGroup, "unprocessedEventThreshold": "15", "eventHubName": testEventHubName, "eventHubNamespace": testEventHubNamespace, "cloud": "private", "activeDirectoryEndpoint": activeDirectoryEndpoint}, true}, + // metadata with private cloud missing service bus endpoint suffix and active directory endpoint + {map[string]string{"storageConnectionFromEnv": storageConnectionSetting, "consumerGroup": eventHubConsumerGroup, "unprocessedEventThreshold": "15", "eventHubName": testEventHubName, + "eventHubNamespace": testEventHubNamespace, "cloud": "private", "eventHubResourceURL": eventHubResourceURL}, true}, // properly formed metadata with private cloud {map[string]string{"storageConnectionFromEnv": storageConnectionSetting, "consumerGroup": eventHubConsumerGroup, "unprocessedEventThreshold": "15", "eventHubName": testEventHubName, - "eventHubNamespace": testEventHubNamespace, "cloud": "private", "endpointSuffix": serviceBusEndpointSuffix, "activeDirectoryEndpoint": activeDirectoryEndpoint}, false}, + "eventHubNamespace": testEventHubNamespace, "cloud": "private", "endpointSuffix": serviceBusEndpointSuffix, "activeDirectoryEndpoint": activeDirectoryEndpoint, "eventHubResourceURL": eventHubResourceURL}, false}, } var eventHubMetricIdentifiers = []eventHubMetricIdentifier{ From 351871f608ede5fdccaa8a4708cfab3adec6abe8 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 15:25:42 +0530 Subject: [PATCH 09/30] Azure AD Workload Identity Support - Azure Blob Storage Scaler. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure/azure_storage.go | 27 ++++++++++++++++++++------- pkg/scalers/azure_blob_scaler.go | 4 ++-- pkg/scalers/azure_blob_scaler_test.go | 16 ++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pkg/scalers/azure/azure_storage.go b/pkg/scalers/azure/azure_storage.go index dd5d7f28296..f68af6d564d 100644 --- a/pkg/scalers/azure/azure_storage.go +++ b/pkg/scalers/azure/azure_storage.go @@ -50,6 +50,11 @@ const ( FileEndpoint ) +const ( + // Azure storage resource is "https://storage.azure.com/" in all cloud environments + storageResource = "https://storage.azure.com/" +) + // Prefix returns prefix for a StorageEndpointType func (e StorageEndpointType) Prefix() string { return [...]string{"BlobEndpoint", "QueueEndpoint", "TableEndpoint", "FileEndpoint"}[e] @@ -77,8 +82,8 @@ func ParseAzureStorageEndpointSuffix(metadata map[string]string, endpointType St // ParseAzureStorageQueueConnection parses queue connection string and returns credential and resource url func ParseAzureStorageQueueConnection(ctx context.Context, httpClient util.HTTPDoer, podIdentity kedav1alpha1.PodIdentityProvider, connectionString, accountName, endpointSuffix string) (azqueue.Credential, *url.URL, error) { switch podIdentity { - case kedav1alpha1.PodIdentityProviderAzure: - token, endpoint, err := parseAcessTokenAndEndpoint(ctx, httpClient, accountName, endpointSuffix) + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: + token, endpoint, err := parseAcessTokenAndEndpoint(ctx, httpClient, accountName, endpointSuffix, podIdentity) if err != nil { return nil, nil, err } @@ -105,8 +110,8 @@ func ParseAzureStorageQueueConnection(ctx context.Context, httpClient util.HTTPD // ParseAzureStorageBlobConnection parses blob connection string and returns credential and resource url func ParseAzureStorageBlobConnection(ctx context.Context, httpClient util.HTTPDoer, podIdentity kedav1alpha1.PodIdentityProvider, connectionString, accountName, endpointSuffix string) (azblob.Credential, *url.URL, error) { switch podIdentity { - case kedav1alpha1.PodIdentityProviderAzure: - token, endpoint, err := parseAcessTokenAndEndpoint(ctx, httpClient, accountName, endpointSuffix) + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: + token, endpoint, err := parseAcessTokenAndEndpoint(ctx, httpClient, accountName, endpointSuffix, podIdentity) if err != nil { return nil, nil, err } @@ -187,9 +192,17 @@ func parseAzureStorageConnectionString(connectionString string, endpointType Sto return u, name, key, nil } -func parseAcessTokenAndEndpoint(ctx context.Context, httpClient util.HTTPDoer, accountName string, endpointSuffix string) (string, *url.URL, error) { - // Azure storage resource is "https://storage.azure.com/" in all cloud environments - token, err := GetAzureADPodIdentityToken(ctx, httpClient, "https://storage.azure.com/") +func parseAcessTokenAndEndpoint(ctx context.Context, httpClient util.HTTPDoer, accountName string, endpointSuffix string, + podIdentity kedav1alpha1.PodIdentityProvider) (string, *url.URL, error) { + var token AADToken + var err error + + if podIdentity == kedav1alpha1.PodIdentityProviderAzure { + token, err = GetAzureADPodIdentityToken(ctx, httpClient, storageResource) + } else { + token, err = GetAzureADWorkloadIdentityToken(ctx, storageResource) + } + if err != nil { return "", nil, err } diff --git a/pkg/scalers/azure_blob_scaler.go b/pkg/scalers/azure_blob_scaler.go index 89382c21e37..6f46e2f5b8f 100644 --- a/pkg/scalers/azure_blob_scaler.go +++ b/pkg/scalers/azure_blob_scaler.go @@ -153,8 +153,8 @@ func parseAzureBlobMetadata(config *ScalerConfig) (*azure.BlobMetadata, kedav1al if len(meta.Connection) == 0 { return nil, "", fmt.Errorf("no connection setting given") } - case kedav1alpha1.PodIdentityProviderAzure: - // If the Use AAD Pod Identity is present then check account name + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: + // If the Use AAD Pod Identity / Workload Identity is present then check account name if val, ok := config.TriggerMetadata["accountName"]; ok && val != "" { meta.AccountName = val } else { diff --git a/pkg/scalers/azure_blob_scaler_test.go b/pkg/scalers/azure_blob_scaler_test.go index 33bf3b6871d..5191f53f078 100644 --- a/pkg/scalers/azure_blob_scaler_test.go +++ b/pkg/scalers/azure_blob_scaler_test.go @@ -69,6 +69,22 @@ var testAzBlobMetadata = []parseAzBlobMetadataTestData{ {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "Private", "endpointSuffix": ""}, true, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, // podIdentity = azure with endpoint suffix and no cloud {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "", "endpointSuffix": "ignored"}, false, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, + // podIdentity = azure-workload with account name + {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container"}, false, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload without account name + {map[string]string{"accountName": "", "blobContainerName": "sample_container"}, true, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload without blob container name + {map[string]string{"accountName": "sample_acc", "blobContainerName": ""}, true, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with cloud + {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "AzureGermanCloud"}, false, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with invalid cloud + {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "InvalidCloud"}, true, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with private cloud and endpoint suffix + {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "Private", "endpointSuffix": "queue.core.private.cloud"}, false, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with private cloud and no endpoint suffix + {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "Private", "endpointSuffix": ""}, true, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with endpoint suffix and no cloud + {map[string]string{"accountName": "sample_acc", "blobContainerName": "sample_container", "cloud": "", "endpointSuffix": "ignored"}, false, testAzBlobResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, // connection from authParams {map[string]string{"blobContainerName": "sample_container", "blobCount": "5"}, false, testAzBlobResolvedEnv, map[string]string{"connection": "value"}, kedav1alpha1.PodIdentityProviderNone}, // with globPattern From ce75346703d1149f5bf9e8a66077f7eba5a239ae Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 16:01:59 +0530 Subject: [PATCH 10/30] Azure AD Workload Identity Support - Azure Storage Queue Scaler. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure_queue_scaler.go | 2 +- pkg/scalers/azure_queue_scaler_test.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/scalers/azure_queue_scaler.go b/pkg/scalers/azure_queue_scaler.go index a1c66d54ed3..4189da95b97 100644 --- a/pkg/scalers/azure_queue_scaler.go +++ b/pkg/scalers/azure_queue_scaler.go @@ -128,7 +128,7 @@ func parseAzureQueueMetadata(config *ScalerConfig) (*azureQueueMetadata, kedav1a if len(meta.connection) == 0 { return nil, "", fmt.Errorf("no connection setting given") } - case kedav1alpha1.PodIdentityProviderAzure: + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: // If the Use AAD Pod Identity is present then check account name if val, ok := config.TriggerMetadata["accountName"]; ok && val != "" { meta.accountName = val diff --git a/pkg/scalers/azure_queue_scaler_test.go b/pkg/scalers/azure_queue_scaler_test.go index 45a8cca6fcf..d8aa2fe5ff4 100644 --- a/pkg/scalers/azure_queue_scaler_test.go +++ b/pkg/scalers/azure_queue_scaler_test.go @@ -73,6 +73,22 @@ var testAzQueueMetadata = []parseAzQueueMetadataTestData{ {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "Private", "endpointSuffix": ""}, true, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, // podIdentity = azure with endpoint suffix and no cloud {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "", "endpointSuffix": "ignored"}, false, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzure}, + // podIdentity = azure-workload with account name + {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue"}, false, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload without account name + {map[string]string{"accountName": "", "queueName": "sample_queue"}, true, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload without queue name + {map[string]string{"accountName": "sample_acc", "queueName": ""}, true, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with cloud + {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "AzurePublicCloud"}, false, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with invalid cloud + {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "InvalidCloud"}, true, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with private cloud and endpoint suffix + {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "Private", "endpointSuffix": "queue.core.private.cloud"}, false, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with private cloud and no endpoint suffix + {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "Private", "endpointSuffix": ""}, true, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, + // podIdentity = azure-workload with endpoint suffix and no cloud + {map[string]string{"accountName": "sample_acc", "queueName": "sample_queue", "cloud": "", "endpointSuffix": "ignored"}, false, testAzQueueResolvedEnv, map[string]string{}, kedav1alpha1.PodIdentityProviderAzureWorkload}, // connection from authParams {map[string]string{"queueName": "sample", "queueLength": "5"}, false, testAzQueueResolvedEnv, map[string]string{"connection": "value"}, kedav1alpha1.PodIdentityProviderNone}, } From 6b97affb53effb2962d8eeb89ad3ca0aa8ec67e1 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 18:16:49 +0530 Subject: [PATCH 11/30] Use 'PodIdentityProvider' instead of 'strings', and 'switch' instead of 'if' for consistency. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure/azure_data_explorer.go | 8 ++--- pkg/scalers/azure/azure_data_explorer_test.go | 21 ++++++------- pkg/scalers/azure/azure_eventhub.go | 27 +++++++++------- pkg/scalers/azure/azure_eventhub_test.go | 15 +++++++++ pkg/scalers/azure/azure_storage.go | 5 +-- pkg/scalers/azure_data_explorer_scaler.go | 2 +- pkg/scalers/azure_eventhub_scaler_test.go | 5 +-- pkg/scalers/azure_log_analytics_scaler.go | 31 ++++++++++--------- pkg/scalers/azure_monitor_scaler.go | 1 + 9 files changed, 69 insertions(+), 46 deletions(-) diff --git a/pkg/scalers/azure/azure_data_explorer.go b/pkg/scalers/azure/azure_data_explorer.go index 7899bd7fbb0..3bc5d1bc840 100644 --- a/pkg/scalers/azure/azure_data_explorer.go +++ b/pkg/scalers/azure/azure_data_explorer.go @@ -37,7 +37,7 @@ type DataExplorerMetadata struct { DatabaseName string Endpoint string MetricName string - PodIdentity string + PodIdentity kedav1alpha1.PodIdentityProvider Query string TenantID string Threshold int64 @@ -69,7 +69,7 @@ func getDataExplorerAuthConfig(ctx context.Context, metadata *DataExplorerMetada var authConfig auth.AuthorizerConfig switch metadata.PodIdentity { - case "", string(kedav1alpha1.PodIdentityProviderNone): + case "", kedav1alpha1.PodIdentityProviderNone: if metadata.ClientID != "" && metadata.ClientSecret != "" && metadata.TenantID != "" { config := auth.NewClientCredentialsConfig(metadata.ClientID, metadata.ClientSecret, metadata.TenantID) config.Resource = metadata.Endpoint @@ -79,14 +79,14 @@ func getDataExplorerAuthConfig(ctx context.Context, metadata *DataExplorerMetada authConfig = config return authConfig, nil } - case string(kedav1alpha1.PodIdentityProviderAzure): + case kedav1alpha1.PodIdentityProviderAzure: config := auth.NewMSIConfig() config.Resource = metadata.Endpoint azureDataExplorerLogger.V(1).Info("Creating Azure Data Explorer Client using Pod Identity") authConfig = config return authConfig, nil - case string(kedav1alpha1.PodIdentityProviderAzureWorkload): + case kedav1alpha1.PodIdentityProviderAzureWorkload: azureDataExplorerLogger.V(1).Info("Creating Azure Data Explorer Client using Workload Identity") authConfig = NewAzureADWorkloadIdentityConfig(ctx, metadata.Endpoint) return authConfig, nil diff --git a/pkg/scalers/azure/azure_data_explorer_test.go b/pkg/scalers/azure/azure_data_explorer_test.go index 6233e1bbd9a..2b459d3378a 100644 --- a/pkg/scalers/azure/azure_data_explorer_test.go +++ b/pkg/scalers/azure/azure_data_explorer_test.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/azure-kusto-go/kusto/data/table" "github.com/Azure/azure-kusto-go/kusto/data/types" "github.com/Azure/azure-kusto-go/kusto/data/value" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) type testExtractDataExplorerMetricValue struct { @@ -37,14 +38,12 @@ type testGetDataExplorerAuthConfig struct { } var ( - clientID = "test_client_id" - rowName = "result" - rowType types.Column = "long" - rowValue int64 = 3 - podIdentity = "azure" - workloadIdentity = "azure-workload" - secret = "test_secret" - tenantID = "test_tenant_id" + clientID = "test_client_id" + rowName = "result" + rowType types.Column = "long" + rowValue int64 = 3 + secret = "test_secret" + tenantID = "test_tenant_id" ) var testExtractDataExplorerMetricValues = []testExtractDataExplorerMetricValue{ @@ -66,9 +65,9 @@ var testGetDataExplorerAuthConfigs = []testGetDataExplorerAuthConfig{ // Auth with aad app - pass {testMetadata: &DataExplorerMetadata{ClientID: clientID, ClientSecret: secret, TenantID: tenantID}, isError: false}, // Auth with podIdentity - pass - {testMetadata: &DataExplorerMetadata{PodIdentity: podIdentity}, isError: false}, - // Auth with podIdentity - pass - {testMetadata: &DataExplorerMetadata{PodIdentity: workloadIdentity}, isError: false}, + {testMetadata: &DataExplorerMetadata{PodIdentity: kedav1alpha1.PodIdentityProviderAzure}, isError: false}, + // Auth with workload identity - pass + {testMetadata: &DataExplorerMetadata{PodIdentity: kedav1alpha1.PodIdentityProviderAzureWorkload}, isError: false}, // Empty metadata - fail {testMetadata: &DataExplorerMetadata{}, isError: true}, // Empty tenantID - fail diff --git a/pkg/scalers/azure/azure_eventhub.go b/pkg/scalers/azure/azure_eventhub.go index 67f671e6c26..a9deab48835 100644 --- a/pkg/scalers/azure/azure_eventhub.go +++ b/pkg/scalers/azure/azure_eventhub.go @@ -34,19 +34,18 @@ const ( // GetEventHubClient returns eventhub client func GetEventHubClient(ctx context.Context, info EventHubInfo) (*eventhub.Hub, error) { - // The user wants to use a connectionstring, not a pod identity - if info.PodIdentity == "" || info.PodIdentity == kedav1alpha1.PodIdentityProviderNone { + switch info.PodIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: + // The user wants to use a connectionstring, not a pod identity hub, err := eventhub.NewHubFromConnectionString(info.EventHubConnection) if err != nil { return nil, fmt.Errorf("failed to create hub client: %s", err) } return hub, nil - } - - env := azure.Environment{ActiveDirectoryEndpoint: info.ActiveDirectoryEndpoint, ServiceBusEndpointSuffix: info.ServiceBusEndpointSuffix} - hubEnvOptions := eventhub.HubWithEnvironment(env) - if info.PodIdentity == kedav1alpha1.PodIdentityProviderAzure { - // Since there is no connectionstring, then user wants to use pod identity + case kedav1alpha1.PodIdentityProviderAzure: + env := azure.Environment{ActiveDirectoryEndpoint: info.ActiveDirectoryEndpoint, ServiceBusEndpointSuffix: info.ServiceBusEndpointSuffix} + hubEnvOptions := eventhub.HubWithEnvironment(env) + // Since there is no connectionstring, then user wants to use AAD Pod identity // Internally, the JWTProvider will use Managed Service Identity to authenticate if no Service Principal info supplied envJWTProviderOption := aad.JWTProviderWithAzureEnvironment(&env) resourceURLJWTProviderOption := aad.JWTProviderWithResourceURI(info.EventHubResourceURL) @@ -57,12 +56,16 @@ func GetEventHubClient(ctx context.Context, info EventHubInfo) (*eventhub.Hub, e } return nil, aadErr - } + case kedav1alpha1.PodIdentityProviderAzureWorkload: + // User wants to use AAD Workload Identity + env := azure.Environment{ActiveDirectoryEndpoint: info.ActiveDirectoryEndpoint, ServiceBusEndpointSuffix: info.ServiceBusEndpointSuffix} + hubEnvOptions := eventhub.HubWithEnvironment(env) + provider := NewADWorkloadIdentityTokenProvider(ctx, info.EventHubResourceURL) - // Workload Identity case - provider := NewADWorkloadIdentityTokenProvider(ctx, info.EventHubResourceURL) + return eventhub.NewHub(info.Namespace, info.EventHubName, provider, hubEnvOptions) + } - return eventhub.NewHub(info.Namespace, info.EventHubName, provider, hubEnvOptions) + return nil, fmt.Errorf("event hub does not support pod identity %v", info.PodIdentity) } // ParseAzureEventHubConnectionString parses Event Hub connection string into (namespace, name) diff --git a/pkg/scalers/azure/azure_eventhub_test.go b/pkg/scalers/azure/azure_eventhub_test.go index d58e60ea269..d0e26f37039 100644 --- a/pkg/scalers/azure/azure_eventhub_test.go +++ b/pkg/scalers/azure/azure_eventhub_test.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-storage-blob-go/azblob" "github.com/go-playground/assert/v2" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) // Add a valid Storage account connection string here @@ -256,12 +257,19 @@ func TestShouldParseCheckpointForFunctionWithPodIdentity(t *testing.T) { EventHubName: "hub-test", EventHubConsumerGroup: "$Default", ServiceBusEndpointSuffix: "servicebus.windows.net", + PodIdentity: kedav1alpha1.PodIdentityProviderAzure, } cp := newCheckpointer(eventHubInfo, "0") url, _ := cp.resolvePath(eventHubInfo) assert.Equal(t, url.Path, "/azure-webjobs-eventhub/eventhubnamespace.servicebus.windows.net/hub-test/$Default/0") + + eventHubInfo.PodIdentity = kedav1alpha1.PodIdentityProviderAzureWorkload + cp = newCheckpointer(eventHubInfo, "0") + url, _ = cp.resolvePath(eventHubInfo) + + assert.Equal(t, url.Path, "/azure-webjobs-eventhub/eventhubnamespace.servicebus.windows.net/hub-test/$Default/0") } func TestShouldParseCheckpointForFunctionWithCheckpointStrategyAndPodIdentity(t *testing.T) { @@ -271,12 +279,19 @@ func TestShouldParseCheckpointForFunctionWithCheckpointStrategyAndPodIdentity(t EventHubConsumerGroup: "$Default", ServiceBusEndpointSuffix: "servicebus.windows.net", CheckpointStrategy: "azureFunction", + PodIdentity: kedav1alpha1.PodIdentityProviderAzure, } cp := newCheckpointer(eventHubInfo, "0") url, _ := cp.resolvePath(eventHubInfo) assert.Equal(t, url.Path, "/azure-webjobs-eventhub/eventhubnamespace.servicebus.windows.net/hub-test/$Default/0") + + eventHubInfo.PodIdentity = kedav1alpha1.PodIdentityProviderAzureWorkload + cp = newCheckpointer(eventHubInfo, "0") + url, _ = cp.resolvePath(eventHubInfo) + + assert.Equal(t, url.Path, "/azure-webjobs-eventhub/eventhubnamespace.servicebus.windows.net/hub-test/$Default/0") } func TestShouldParseCheckpointForDefault(t *testing.T) { diff --git a/pkg/scalers/azure/azure_storage.go b/pkg/scalers/azure/azure_storage.go index f68af6d564d..b649baf9172 100644 --- a/pkg/scalers/azure/azure_storage.go +++ b/pkg/scalers/azure/azure_storage.go @@ -197,9 +197,10 @@ func parseAcessTokenAndEndpoint(ctx context.Context, httpClient util.HTTPDoer, a var token AADToken var err error - if podIdentity == kedav1alpha1.PodIdentityProviderAzure { + switch podIdentity { + case kedav1alpha1.PodIdentityProviderAzure: token, err = GetAzureADPodIdentityToken(ctx, httpClient, storageResource) - } else { + case kedav1alpha1.PodIdentityProviderAzureWorkload: token, err = GetAzureADWorkloadIdentityToken(ctx, storageResource) } diff --git a/pkg/scalers/azure_data_explorer_scaler.go b/pkg/scalers/azure_data_explorer_scaler.go index 08935ed68fc..38952bfab3a 100644 --- a/pkg/scalers/azure_data_explorer_scaler.go +++ b/pkg/scalers/azure_data_explorer_scaler.go @@ -133,7 +133,7 @@ func parseAzureDataExplorerAuthParams(config *ScalerConfig) (*azure.DataExplorer switch config.PodIdentity { case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: - metadata.PodIdentity = string(config.PodIdentity) + metadata.PodIdentity = config.PodIdentity case "", kedav1alpha1.PodIdentityProviderNone: dataExplorerLogger.V(1).Info("Pod Identity is not provided. Trying to resolve clientId, clientSecret and tenantId.") diff --git a/pkg/scalers/azure_eventhub_scaler_test.go b/pkg/scalers/azure_eventhub_scaler_test.go index 33d3b54c067..74399ee8c0b 100644 --- a/pkg/scalers/azure_eventhub_scaler_test.go +++ b/pkg/scalers/azure_eventhub_scaler_test.go @@ -11,6 +11,7 @@ import ( eventhub "github.com/Azure/azure-event-hubs-go/v3" "github.com/Azure/azure-storage-blob-go/azblob" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/scalers/azure" ) @@ -114,7 +115,7 @@ func TestParseEventHubMetadata(t *testing.T) { } for _, testData := range parseEventHubMetadataDatasetWithPodIdentity { - _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: "azure"}) + _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: kedav1alpha1.PodIdentityProviderAzure}) if err != nil && !testData.isError { t.Errorf("Expected success but got error: %s", err) @@ -125,7 +126,7 @@ func TestParseEventHubMetadata(t *testing.T) { } for _, testData := range parseEventHubMetadataDatasetWithPodIdentity { - _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: "azure-workload"}) + _, err := parseAzureEventHubMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, ResolvedEnv: sampleEventHubResolvedEnv, AuthParams: map[string]string{}, PodIdentity: kedav1alpha1.PodIdentityProviderAzureWorkload}) if err != nil && !testData.isError { t.Errorf("Expected success but got error: %s", err) diff --git a/pkg/scalers/azure_log_analytics_scaler.go b/pkg/scalers/azure_log_analytics_scaler.go index e83d1963103..45220eeedef 100644 --- a/pkg/scalers/azure_log_analytics_scaler.go +++ b/pkg/scalers/azure_log_analytics_scaler.go @@ -65,7 +65,7 @@ type azureLogAnalyticsMetadata struct { clientID string clientSecret string workspaceID string - podIdentity string + podIdentity kedav1alpha1.PodIdentityProvider query string threshold int64 metricName string // Custom metric name for trigger @@ -168,7 +168,7 @@ func parseAzureLogAnalyticsMetadata(config *ScalerConfig) (*azureLogAnalyticsMet meta.podIdentity = "" case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: - meta.podIdentity = string(config.PodIdentity) + meta.podIdentity = config.PodIdentity default: return nil, fmt.Errorf("error parsing metadata. Details: Log Analytics Scaler doesn't support pod identity %s", config.PodIdentity) } @@ -335,10 +335,11 @@ func (s *azureLogAnalyticsScaler) getAccessToken(ctx context.Context) (tokenData currentTimeSec := time.Now().Unix() tokenInfo := tokenData{} - if s.metadata.podIdentity == "" { + switch s.metadata.podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: tokenInfo, _ = getTokenFromCache(s.metadata.clientID, s.metadata.clientSecret) - } else { - tokenInfo, _ = getTokenFromCache(s.metadata.podIdentity, s.metadata.podIdentity) + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: + tokenInfo, _ = getTokenFromCache(string(s.metadata.podIdentity), string(s.metadata.podIdentity)) } if currentTimeSec+30 > tokenInfo.ExpiresOn { @@ -347,12 +348,13 @@ func (s *azureLogAnalyticsScaler) getAccessToken(ctx context.Context) (tokenData return tokenData{}, err } - if s.metadata.podIdentity == "" { + switch s.metadata.podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: logAnalyticsLog.V(1).Info("Token for Service Principal has been refreshed", "clientID", s.metadata.clientID, "scaler name", s.name, "namespace", s.namespace) _ = setTokenInCache(s.metadata.clientID, s.metadata.clientSecret, newTokenInfo) - } else { + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: logAnalyticsLog.V(1).Info("Token for Pod Identity has been refreshed", "type", s.metadata.podIdentity, "scaler name", s.name, "namespace", s.namespace) - _ = setTokenInCache(s.metadata.podIdentity, s.metadata.podIdentity, newTokenInfo) + _ = setTokenInCache(string(s.metadata.podIdentity), string(s.metadata.podIdentity), newTokenInfo) } return newTokenInfo, nil @@ -375,12 +377,13 @@ func (s *azureLogAnalyticsScaler) executeQuery(ctx context.Context, query string return metricsData{}, err } - if s.metadata.podIdentity == "" { + switch s.metadata.podIdentity { + case "", kedav1alpha1.PodIdentityProviderNone: logAnalyticsLog.V(1).Info("Token for Service Principal has been refreshed", "clientID", s.metadata.clientID, "scaler name", s.name, "namespace", s.namespace) _ = setTokenInCache(s.metadata.clientID, s.metadata.clientSecret, tokenInfo) - } else { + case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: logAnalyticsLog.V(1).Info("Token for Pod Identity has been refreshed", "type", s.metadata.podIdentity, "scaler name", s.name, "namespace", s.namespace) - _ = setTokenInCache(s.metadata.podIdentity, s.metadata.podIdentity, tokenInfo) + _ = setTokenInCache(string(s.metadata.podIdentity), string(s.metadata.podIdentity), tokenInfo) } if err == nil { @@ -501,7 +504,7 @@ func (s *azureLogAnalyticsScaler) getAuthorizationToken(ctx context.Context) (to var tokenInfo tokenData switch s.metadata.podIdentity { - case string(kedav1alpha1.PodIdentityProviderAzureWorkload): + case kedav1alpha1.PodIdentityProviderAzureWorkload: aadToken, err := azure.GetAzureADWorkloadIdentityToken(ctx, s.metadata.logAnalyticsResourceURL) if err != nil { return tokenData{}, nil @@ -521,9 +524,9 @@ func (s *azureLogAnalyticsScaler) getAuthorizationToken(ctx context.Context) (to } return tokenInfo, nil - case "", string(kedav1alpha1.PodIdentityProviderNone): + case "", kedav1alpha1.PodIdentityProviderNone: body, statusCode, err = s.executeAADApicall(ctx) - case string(kedav1alpha1.PodIdentityProviderAzure): + case kedav1alpha1.PodIdentityProviderAzure: body, statusCode, err = s.executeIMDSApicall(ctx) } diff --git a/pkg/scalers/azure_monitor_scaler.go b/pkg/scalers/azure_monitor_scaler.go index 2666afc464a..37f1f3fcc69 100644 --- a/pkg/scalers/azure_monitor_scaler.go +++ b/pkg/scalers/azure_monitor_scaler.go @@ -193,6 +193,7 @@ func parseAzurePodIdentityParams(config *ScalerConfig) (clientID string, clientP return "", "", fmt.Errorf("no activeDirectoryClientPassword given") } case kedav1alpha1.PodIdentityProviderAzure, kedav1alpha1.PodIdentityProviderAzureWorkload: + // no params required to be parsed default: return "", "", fmt.Errorf("azure Monitor doesn't support pod identity %s", config.PodIdentity) } From 77a31fcfca5f03d4e1f7fa3e7f3981db311838b2 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 22 Apr 2022 18:58:12 +0530 Subject: [PATCH 12/30] Refresh token only if expired. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure/azure_aad_auth.go | 21 +++++++++++-------- .../azure/azure_aad_workload_identity.go | 14 +++++++++---- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/pkg/scalers/azure/azure_aad_auth.go b/pkg/scalers/azure/azure_aad_auth.go index cb1cacf52a5..a96084c08b7 100644 --- a/pkg/scalers/azure/azure_aad_auth.go +++ b/pkg/scalers/azure/azure_aad_auth.go @@ -16,15 +16,18 @@ limitations under the License. package azure +import "time" + // AADToken is the token from Azure AD type AADToken struct { - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - ExpiresIn string `json:"expires_in"` - ExpiresOn string `json:"expires_on"` - NotBefore string `json:"not_before"` - Resource string `json:"resource"` - TokenType string `json:"token_type"` - GrantedScopes []string `json:"grantedScopes"` - DeclinedScopes []string `json:"DeclinedScopes"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn string `json:"expires_in"` + ExpiresOn string `json:"expires_on"` + ExpiresOnTimeObject time.Time `json:"expires_on_object"` + NotBefore string `json:"not_before"` + Resource string `json:"resource"` + TokenType string `json:"token_type"` + GrantedScopes []string `json:"grantedScopes"` + DeclinedScopes []string `json:"DeclinedScopes"` } diff --git a/pkg/scalers/azure/azure_aad_workload_identity.go b/pkg/scalers/azure/azure_aad_workload_identity.go index 92dcfab210f..d9bfd321cdf 100644 --- a/pkg/scalers/azure/azure_aad_workload_identity.go +++ b/pkg/scalers/azure/azure_aad_workload_identity.go @@ -22,6 +22,7 @@ import ( "os" "strconv" "strings" + "time" amqpAuth "github.com/Azure/azure-amqp-common-go/v3/auth" "github.com/Azure/go-autorest/autorest" @@ -75,10 +76,11 @@ func GetAzureADWorkloadIdentityToken(ctx context.Context, resource string) (AADT } return AADToken{ - AccessToken: result.AccessToken, - ExpiresOn: strconv.FormatInt(result.ExpiresOn.Unix(), 10), - GrantedScopes: result.GrantedScopes, - DeclinedScopes: result.DeclinedScopes, + AccessToken: result.AccessToken, + ExpiresOn: strconv.FormatInt(result.ExpiresOn.Unix(), 10), + ExpiresOnTimeObject: result.ExpiresOn, + GrantedScopes: result.GrantedScopes, + DeclinedScopes: result.DeclinedScopes, }, nil } @@ -133,6 +135,10 @@ func (wiTokenProvider *ADWorkloadIdentityTokenProvider) OAuthToken() string { // Refresh is for implementing the adal.Refresher interface func (wiTokenProvider *ADWorkloadIdentityTokenProvider) Refresh() error { + if time.Now().Before(wiTokenProvider.aadToken.ExpiresOnTimeObject) { + return nil + } + aadToken, err := GetAzureADWorkloadIdentityToken(wiTokenProvider.ctx, wiTokenProvider.Resource) if err != nil { return err From a529e0c8b591371cfa4d0b013a8b14d41af50699 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Sat, 23 Apr 2022 01:00:36 +0530 Subject: [PATCH 13/30] Minor refactoring. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure_log_analytics_scaler.go | 2 +- pkg/scalers/azure_servicebus_scaler.go | 3 +-- pkg/scaling/resolver/azure_keyvault_handler.go | 18 +++++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/scalers/azure_log_analytics_scaler.go b/pkg/scalers/azure_log_analytics_scaler.go index 45220eeedef..96b9243b6f2 100644 --- a/pkg/scalers/azure_log_analytics_scaler.go +++ b/pkg/scalers/azure_log_analytics_scaler.go @@ -510,7 +510,7 @@ func (s *azureLogAnalyticsScaler) getAuthorizationToken(ctx context.Context) (to return tokenData{}, nil } - expiresOn, err := strconv.ParseInt(aadToken.ExpiresOn, 10, 64) + expiresOn := aadToken.ExpiresOnTimeObject.Unix() if err != nil { return tokenData{}, nil } diff --git a/pkg/scalers/azure_servicebus_scaler.go b/pkg/scalers/azure_servicebus_scaler.go index de739cc27b2..6f434be3bba 100755 --- a/pkg/scalers/azure_servicebus_scaler.go +++ b/pkg/scalers/azure_servicebus_scaler.go @@ -242,8 +242,7 @@ func (a azureTokenProvider) GetToken(uri string) (*auth.Token, error) { case kedav1alpha1.PodIdentityProviderAzure: token, err = azure.GetAzureADPodIdentityToken(ctx, a.httpClient, serviceBusResource) case kedav1alpha1.PodIdentityProviderAzureWorkload: - scopedResource := fmt.Sprintf("%s%s", serviceBusResource, ".default") - token, err = azure.GetAzureADWorkloadIdentityToken(ctx, scopedResource) + token, err = azure.GetAzureADWorkloadIdentityToken(ctx, serviceBusResource) default: err = fmt.Errorf("unknown pod identity provider") } diff --git a/pkg/scaling/resolver/azure_keyvault_handler.go b/pkg/scaling/resolver/azure_keyvault_handler.go index fb82b626c4d..3dde2437834 100644 --- a/pkg/scaling/resolver/azure_keyvault_handler.go +++ b/pkg/scaling/resolver/azure_keyvault_handler.go @@ -50,7 +50,7 @@ func (vh *AzureKeyVaultHandler) Initialize(ctx context.Context, client client.Cl return err } - authConfig := vh.getAuthConfig(ctx, client, logger, triggerNamespace, keyvaultResourceURL, activeDirectoryEndpoint) + authConfig, err := vh.getAuthConfig(ctx, client, logger, triggerNamespace, keyvaultResourceURL, activeDirectoryEndpoint) if err != nil { return err } @@ -103,7 +103,7 @@ func (vh *AzureKeyVaultHandler) getPropertiesForCloud() (string, string, error) } func (vh *AzureKeyVaultHandler) getAuthConfig(ctx context.Context, client client.Client, logger logr.Logger, - triggerNamespace, keyVaultResourceURL, activeDirectoryEndpoint string) auth.AuthorizerConfig { + triggerNamespace, keyVaultResourceURL, activeDirectoryEndpoint string) (auth.AuthorizerConfig, error) { switch vh.podIdentity { case "", kedav1alpha1.PodIdentityProviderNone: clientID := vh.vault.Credentials.ClientID @@ -113,19 +113,23 @@ func (vh *AzureKeyVaultHandler) getAuthConfig(ctx context.Context, client client clientSecretKey := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Key clientSecret := resolveAuthSecret(ctx, client, logger, clientSecretName, triggerNamespace, clientSecretKey) + if clientID == "" || tenantID == "" || clientSecret == "" { + return nil, fmt.Errorf("clientID, tenantID and clientSecret are expected when not using a pod identity provider") + } + config := auth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) config.Resource = keyVaultResourceURL config.AADEndpoint = activeDirectoryEndpoint - return config + return config, nil case kedav1alpha1.PodIdentityProviderAzure: config := auth.NewMSIConfig() config.Resource = keyVaultResourceURL - return config + return config, nil case kedav1alpha1.PodIdentityProviderAzureWorkload: - return azure.NewAzureADWorkloadIdentityConfig(ctx, keyVaultResourceURL) + return azure.NewAzureADWorkloadIdentityConfig(ctx, keyVaultResourceURL), nil + default: + return nil, fmt.Errorf("key vault does not support pod identity provider - %s", vh.podIdentity) } - - return nil } From 8a731d1c11140dd837910ddb7b04c4ccef00a5ee Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Mon, 25 Apr 2022 15:45:22 +0530 Subject: [PATCH 14/30] Fix application insight e2e test. Signed-off-by: Vighnesh Shenoy --- .github/workflows/main-build.yml | 1 + .github/workflows/nightly-e2e.yml | 1 + .github/workflows/pr-e2e.yml | 1 + tests/scalers/azure-app-insights.test.ts | 12 ++++++++---- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index b3ac7f850b8..a0009c89df0 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -92,6 +92,7 @@ jobs: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY: ${{ secrets.AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY }} + AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_DATA_EXPLORER_DB: ${{ secrets.AZURE_DATA_EXPLORER_DB }} AZURE_DATA_EXPLORER_ENDPOINT: ${{ secrets.AZURE_DATA_EXPLORER_ENDPOINT }} AZURE_DEVOPS_BUILD_DEFINITON_ID: ${{ secrets.AZURE_DEVOPS_BUILD_DEFINITON_ID }} diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 09d2c5e4e24..889800b7b3c 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -23,6 +23,7 @@ jobs: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY: ${{ secrets.AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY }} + AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_DATA_EXPLORER_DB: ${{ secrets.AZURE_DATA_EXPLORER_DB }} AZURE_DATA_EXPLORER_ENDPOINT: ${{ secrets.AZURE_DATA_EXPLORER_ENDPOINT }} AZURE_DEVOPS_BUILD_DEFINITON_ID: ${{ secrets.AZURE_DEVOPS_BUILD_DEFINITON_ID }} diff --git a/.github/workflows/pr-e2e.yml b/.github/workflows/pr-e2e.yml index 1f608b9d280..447f0b2919d 100644 --- a/.github/workflows/pr-e2e.yml +++ b/.github/workflows/pr-e2e.yml @@ -96,6 +96,7 @@ jobs: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY: ${{ secrets.AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY }} + AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_DATA_EXPLORER_DB: ${{ secrets.AZURE_DATA_EXPLORER_DB }} AZURE_DATA_EXPLORER_ENDPOINT: ${{ secrets.AZURE_DATA_EXPLORER_ENDPOINT }} AZURE_DEVOPS_BUILD_DEFINITON_ID: ${{ secrets.AZURE_DEVOPS_BUILD_DEFINITON_ID }} diff --git a/tests/scalers/azure-app-insights.test.ts b/tests/scalers/azure-app-insights.test.ts index 4dbe1ade8f4..dc69e91091e 100644 --- a/tests/scalers/azure-app-insights.test.ts +++ b/tests/scalers/azure-app-insights.test.ts @@ -12,6 +12,7 @@ const namespacePrefix = 'azure-ai-test-' const app_insights_app_id = process.env['AZURE_APP_INSIGHTS_APP_ID'] const app_insights_instrumentation_key = process.env['AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY'] const sp_id = process.env['AZURE_SP_APP_ID'] +const app_insights_connection_string = process.env['AZURE_APP_INSIGHTS_CONNECTION_STRING'] const sp_key = process.env['AZURE_SP_KEY'] const sp_tenant = process.env['AZURE_SP_TENANT'] const test_pod_id = process.env['TEST_POD_ID'] == "true" @@ -48,13 +49,15 @@ function sleep(sec: number) { } function set_metric(metric_value, t, test_callback) { - appinsights.setup(app_insights_instrumentation_key).setUseDiskRetryCaching(true) + appinsights.setup(app_insights_connection_string).setUseDiskRetryCaching(true) appinsights.defaultClient.context.tags[appinsights.defaultClient.context.keys.cloudRole] = test_app_insights_role appinsights.defaultClient.trackMetric({name: test_app_insights_metric, value: metric_value}); appinsights.defaultClient.flush({ callback: function(response: string) { - let resp = JSON.parse(response) - t.is(0, resp['errors'].length, `failed to set metric: ${response['errors']}`) + let resp_errors = JSON.parse(response)['errors'] + if (resp_errors != null && resp_errors != undefined) { + t.is(0, resp_errors.length, `failed to set metric: ${JSON.stringify(resp_errors)}`) + } test_callback() } }) @@ -76,7 +79,8 @@ function assert_replicas(t, namespace: string, name: string, replicas: number, w } test.before(t => { - if (!app_insights_app_id || !app_insights_instrumentation_key || !sp_id || !sp_key || !sp_tenant) { + if (!app_insights_app_id || !app_insights_instrumentation_key + || !app_insights_connection_string || !sp_id || !sp_key || !sp_tenant) { t.fail('A required parameters app insights scaler was not resolved') } From a55bb85ca438f8c02c7bfac9c3322603c5ee9b46 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Mon, 25 Apr 2022 19:32:24 +0530 Subject: [PATCH 15/30] Undo workflow .yml changes. Signed-off-by: Vighnesh Shenoy --- .github/workflows/main-build.yml | 1 - .github/workflows/nightly-e2e.yml | 1 - .github/workflows/pr-e2e.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index a0009c89df0..b3ac7f850b8 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -92,7 +92,6 @@ jobs: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY: ${{ secrets.AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY }} - AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_DATA_EXPLORER_DB: ${{ secrets.AZURE_DATA_EXPLORER_DB }} AZURE_DATA_EXPLORER_ENDPOINT: ${{ secrets.AZURE_DATA_EXPLORER_ENDPOINT }} AZURE_DEVOPS_BUILD_DEFINITON_ID: ${{ secrets.AZURE_DEVOPS_BUILD_DEFINITON_ID }} diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 889800b7b3c..09d2c5e4e24 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -23,7 +23,6 @@ jobs: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY: ${{ secrets.AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY }} - AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_DATA_EXPLORER_DB: ${{ secrets.AZURE_DATA_EXPLORER_DB }} AZURE_DATA_EXPLORER_ENDPOINT: ${{ secrets.AZURE_DATA_EXPLORER_ENDPOINT }} AZURE_DEVOPS_BUILD_DEFINITON_ID: ${{ secrets.AZURE_DEVOPS_BUILD_DEFINITON_ID }} diff --git a/.github/workflows/pr-e2e.yml b/.github/workflows/pr-e2e.yml index 447f0b2919d..1f608b9d280 100644 --- a/.github/workflows/pr-e2e.yml +++ b/.github/workflows/pr-e2e.yml @@ -96,7 +96,6 @@ jobs: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY: ${{ secrets.AZURE_APP_INSIGHTS_INSTRUMENTATION_KEY }} - AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} AZURE_DATA_EXPLORER_DB: ${{ secrets.AZURE_DATA_EXPLORER_DB }} AZURE_DATA_EXPLORER_ENDPOINT: ${{ secrets.AZURE_DATA_EXPLORER_ENDPOINT }} AZURE_DEVOPS_BUILD_DEFINITON_ID: ${{ secrets.AZURE_DEVOPS_BUILD_DEFINITON_ID }} From a2bc1d5736f75b3559c086bc6431125075ea9c56 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 27 Apr 2022 22:22:14 +0530 Subject: [PATCH 16/30] Fix import blocks. Signed-off-by: Vighnesh Shenoy --- pkg/scalers/azure/azure_data_explorer_test.go | 1 + pkg/scalers/azure/azure_eventhub_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/scalers/azure/azure_data_explorer_test.go b/pkg/scalers/azure/azure_data_explorer_test.go index 2b459d3378a..a18918927d7 100644 --- a/pkg/scalers/azure/azure_data_explorer_test.go +++ b/pkg/scalers/azure/azure_data_explorer_test.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/azure-kusto-go/kusto/data/table" "github.com/Azure/azure-kusto-go/kusto/data/types" "github.com/Azure/azure-kusto-go/kusto/data/value" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) diff --git a/pkg/scalers/azure/azure_eventhub_test.go b/pkg/scalers/azure/azure_eventhub_test.go index d0e26f37039..2f7f2f30dd3 100644 --- a/pkg/scalers/azure/azure_eventhub_test.go +++ b/pkg/scalers/azure/azure_eventhub_test.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-storage-blob-go/azblob" "github.com/go-playground/assert/v2" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) From 19ed744a8b57c6304050b891b884d702448f48a4 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 27 Apr 2022 22:39:10 +0530 Subject: [PATCH 17/30] Add e2e test for workload identity using service bus scaler. Signed-off-by: Vighnesh Shenoy --- tests/run-all.sh | 7 + ...ervice-bus-queue-workload-identity.test.ts | 211 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 tests/scalers/azure-service-bus-queue-workload-identity.test.ts diff --git a/tests/run-all.sh b/tests/run-all.sh index 47dcc60f970..d49eb9c07a1 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -28,6 +28,13 @@ function run_tests { continue fi + # Disable until workload identity support is added to test environment + # Refer - https://github.com/kedacore/keda/issues/2941 + if [[ $test_case == *workload-identity.test.ts ]] + then + continue + fi + counter=$((counter+1)) ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & pid=$! diff --git a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts new file mode 100644 index 00000000000..35157f9883f --- /dev/null +++ b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts @@ -0,0 +1,211 @@ +import * as sh from "shelljs" +import * as azure from "@azure/service-bus" +import test from "ava" +import { createNamespace, createYamlFile, waitForDeploymentReplicaCount } from "./helpers" + +const connectionString = process.env["AZURE_SERVICE_BUS_CONNECTION_STRING"] +// Format for connection string - +// Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=" +const serviceBusNameSpace = connectionString.split("//")[1].split(".")[0] +const queueName = "sb-queue" + +const testName = "test-azure-service-bus-queue" +const testNamespace = `${testName}-ns` +const secretName = `${testName}-secret` +const deploymentName = `${testName}-deployment` +const triggerAuthName = `${testName}-trigger-auth` +const scaledObjectName = `${testName}-scaled-object` + +test.before(async t => { + if (!connectionString) { + t.fail("AZURE_SERVICE_BUS_CONNECTION_STRING environment variable is required for service bus tests") + } + + sh.config.silent = true + + // Create queue within the Service Bus Namespace + const serviceBusAdminClient = new azure.ServiceBusAdministrationClient(connectionString) + const queueExists = await serviceBusAdminClient.queueExists(queueName) + // Clean up (delete) queue if already exists and create again + if (queueExists) { + await serviceBusAdminClient.deleteQueue(queueName) + } + await serviceBusAdminClient.createQueue(queueName) + + // Create Kubernetes Namespace + createNamespace(testNamespace) + + // Create Secret + const base64ConStr = Buffer.from(connectionString).toString("base64") + const secretFileName = createYamlFile(secretYaml.replace("{{CONNECTION}}", base64ConStr)) + + t.is( + sh.exec(`kubectl apply -f ${secretFileName} -n ${testNamespace}`).code, + 0, + "Creating a secret should work" + ) + + // Create deployment + t.is( + sh.exec(`kubectl apply -f ${createYamlFile(deploymentYaml)} -n ${testNamespace}`).code, + 0, + "Creating a deployment should work" + ) + + // Create trigger auth resource + t.is( + sh.exec(`kubectl apply -f ${createYamlFile(triggerAuthYaml)} -n ${testNamespace}`).code, + 0, + "Creating a trigger authentication resource should work" + ) + + // Create scaled object + t.is( + sh.exec(`kubectl apply -f ${createYamlFile(scaledObjectYaml)} -n ${testNamespace}`).code, + 0, + "Creating a scaled object should work" + ) + + t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), "Replica count should be 0 after 1 minute") +}) + +test.serial("Deployment should scale up with messages on service bus queue", async t => { + // Send messages to service bus queue + const serviceBusClient = new azure.ServiceBusClient(connectionString) + const sender = serviceBusClient.createSender(queueName) + + const messages: azure.ServiceBusMessage[] = [ + {"body": "1"}, + {"body": "2"}, + {"body": "3"}, + {"body": "4"}, + {"body": "5"}, + ] + + await sender.sendMessages(messages) + + await serviceBusClient.close() + + // Scale out when messages available + t.true(await waitForDeploymentReplicaCount(1, deploymentName, testNamespace, 60, 1000), "Replica count should be 1 after 1 minute") +}) + +test.serial("Deployment should scale down with messages on service bus queue", async t => { + // Receive messages from service bus queue + const serviceBusClient = new azure.ServiceBusClient(connectionString) + const receiver = serviceBusClient.createReceiver(queueName) + + var numOfReceivedMessages = 0 + + while (numOfReceivedMessages < 5) { + const messages = await receiver.receiveMessages(10, { + maxWaitTimeInMs: 60 * 1000, + }) + + for (const message of messages) { + await receiver.completeMessage(message) + numOfReceivedMessages += 1 + } + } + + await serviceBusClient.close() + + // Scale down when messages unavailable + t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), "Replica count should be 0 after 1 minute") +}) + +test.after.always("Clean up E2E K8s objects", async t => { + const resources = [ + `scaledobject.keda.sh/${scaledObjectName}`, + `triggerauthentications.keda.sh/${triggerAuthName}`, + `deployments.apps/${deploymentName}`, + `secrets/${secretName}`, + ] + + for (const resource of resources) { + sh.exec(`kubectl delete ${resource} -n ${testNamespace}`) + } + + sh.exec(`kubectl delete ns ${testNamespace}`) + + // Delete queue + const serviceBusAdminClient = new azure.ServiceBusAdministrationClient(connectionString) + const response = await serviceBusAdminClient.deleteQueue(queueName) + t.is( + response._response.status, + 200, + "Queue deletion must succeed" + ) +}) + +// YAML Definitions for Kubernetes resources +// Secret +const secretYaml = +`apiVersion: v1 +kind: Secret +metadata: + name: ${secretName} + namespace: ${testNamespace} +type: Opaque +data: + connection: {{CONNECTION}} +` + +// Deployment +const deploymentYaml = +`apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${deploymentName} + namespace: ${testNamespace} +spec: + replicas: 0 + selector: + matchLabels: + app: ${deploymentName} + template: + metadata: + labels: + app: ${deploymentName} + spec: + containers: + - name: nginx + image: nginx:1.16.1 +` + +// Trigger Authentication +const triggerAuthYaml = +`apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: ${triggerAuthName} + namespace: ${testNamespace} +spec: + podIdentity: + provider: azure-workload +` + +// Scaled Object +const scaledObjectYaml = +`apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: ${scaledObjectName} + namespace: ${testNamespace} + labels: + deploymentName: ${deploymentName} +spec: + scaleTargetRef: + name: ${deploymentName} + pollingInterval: 5 + cooldownPeriod: 10 + minReplicaCount: 0 + maxReplicaCount: 1 + triggers: + - type: azure-servicebus + metadata: + namespace: ${serviceBusNameSpace} + queueName: ${queueName} + authenticationRef: + name: ${triggerAuthName} +` From 1e549df3923d0fa6ebfc45fd1f75c6884e5398b6 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 28 Apr 2022 04:56:36 +0530 Subject: [PATCH 18/30] Remove unused secret from e2e test. Signed-off-by: Vighnesh Shenoy --- ...ervice-bus-queue-workload-identity.test.ts | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts index 35157f9883f..36aacf55796 100644 --- a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts +++ b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts @@ -4,14 +4,13 @@ import test from "ava" import { createNamespace, createYamlFile, waitForDeploymentReplicaCount } from "./helpers" const connectionString = process.env["AZURE_SERVICE_BUS_CONNECTION_STRING"] -// Format for connection string - +// Format for connection string - // Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=" const serviceBusNameSpace = connectionString.split("//")[1].split(".")[0] const queueName = "sb-queue" const testName = "test-azure-service-bus-queue" const testNamespace = `${testName}-ns` -const secretName = `${testName}-secret` const deploymentName = `${testName}-deployment` const triggerAuthName = `${testName}-trigger-auth` const scaledObjectName = `${testName}-scaled-object` @@ -35,16 +34,6 @@ test.before(async t => { // Create Kubernetes Namespace createNamespace(testNamespace) - // Create Secret - const base64ConStr = Buffer.from(connectionString).toString("base64") - const secretFileName = createYamlFile(secretYaml.replace("{{CONNECTION}}", base64ConStr)) - - t.is( - sh.exec(`kubectl apply -f ${secretFileName} -n ${testNamespace}`).code, - 0, - "Creating a secret should work" - ) - // Create deployment t.is( sh.exec(`kubectl apply -f ${createYamlFile(deploymentYaml)} -n ${testNamespace}`).code, @@ -119,7 +108,6 @@ test.after.always("Clean up E2E K8s objects", async t => { `scaledobject.keda.sh/${scaledObjectName}`, `triggerauthentications.keda.sh/${triggerAuthName}`, `deployments.apps/${deploymentName}`, - `secrets/${secretName}`, ] for (const resource of resources) { @@ -139,18 +127,6 @@ test.after.always("Clean up E2E K8s objects", async t => { }) // YAML Definitions for Kubernetes resources -// Secret -const secretYaml = -`apiVersion: v1 -kind: Secret -metadata: - name: ${secretName} - namespace: ${testNamespace} -type: Opaque -data: - connection: {{CONNECTION}} -` - // Deployment const deploymentYaml = `apiVersion: apps/v1 From 1a08f722852d07d50ac82b5be8d2d29205a15eb6 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 28 Apr 2022 20:04:44 +0530 Subject: [PATCH 19/30] Setup and clean up for Azure Workload Identity e2e tests. Signed-off-by: Vighnesh Shenoy --- tests/cleanup.test.ts | 23 +++++++- tests/run-all.sh | 7 --- tests/setup.test.ts | 124 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 134 insertions(+), 20 deletions(-) diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index b779649d6da..9621f844b2d 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -1,14 +1,35 @@ import * as sh from 'shelljs' import test from 'ava' +const workloadIdentityNamespace = "azure-workload-identity-system" +const federatedIdentityCredentialName = "keda-e2e-federated-credential" + test.before('setup shelljs', () => { sh.config.silent = true }) -test('Remove KEDA', t => { +const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT'] + +test.serial('Remove KEDA', t => { let result = sh.exec('(cd .. && make undeploy)') if (result.code !== 0) { t.fail('error removing keda. ' + result) } t.pass('KEDA undeployed successfully using make undeploy command') }) + +test.serial('remove azure workload identity kubernetes components', t => { + + t.is(0, + sh.exec(`helm uninstall workload-identity-webhook --namespace ${workloadIdentityNamespace}`).code, + 'should be able to uninstall workload identity webhook' + ) + + sh.exec(`kubectl delete ns ${workloadIdentityNamespace}`) + + let uri = `https://graph.microsoft.com/beta/applications/${AZURE_AD_OBJECT_ID}/federatedIdentityCredentials/${federatedIdentityCredentialName}` + t.is(0, + sh.exec(`az rest --method DELETE --uri ${uri}`).code, + "should be able to delete federated identity credential" + ) +}) diff --git a/tests/run-all.sh b/tests/run-all.sh index d49eb9c07a1..47dcc60f970 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -28,13 +28,6 @@ function run_tests { continue fi - # Disable until workload identity support is added to test environment - # Refer - https://github.com/kedacore/keda/issues/2941 - if [[ $test_case == *workload-identity.test.ts ]] - then - continue - fi - counter=$((counter+1)) ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & pid=$! diff --git a/tests/setup.test.ts b/tests/setup.test.ts index 650d4e5dbcd..15a32946e18 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -5,6 +5,15 @@ import test from 'ava' const kc = new k8s.KubeConfig() kc.loadFromDefault() +const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_ID'] +const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT'] +const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] +const SERVICE_ACCOUNT_ISSUER = process.env['AZURE_OIDC_ISSUER_URL'] +const SERVICE_ACCOUNT_NAMESPACE = 'keda' +const SERVICE_ACCOUNT_NAME = 'keda-operator' +const workloadIdentityNamespace = "azure-workload-identity-system" +const federatedIdentityCredentialName = "keda-e2e-federated-credential" + test.before('configure shelljs', () => { sh.config.silent = true }) @@ -41,6 +50,99 @@ test.serial('Deploy KEDA', t => { t.pass('KEDA deployed successfully using make deploy command') }) +test.serial('setup helm', t => { + // check if helm is already installed. + let result = sh.exec('helm version') + if(result.code == 0) { + t.pass('helm is already installed. skipping setup') + return + } + t.is(0, sh.exec(`curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3`).code, 'should be able to download helm script') + t.is(0, sh.exec(`chmod 700 get_helm.sh`).code, 'should be able to change helm script permissions') + t.is(0, sh.exec(`./get_helm.sh`).code, 'should be able to download helm') + t.is(0, sh.exec(`helm version`).code, 'should be able to get helm version') +}) + +test.serial('setup and verify azure workload identity kubernetes components', t => { + // check if helm is already installed. + let result = sh.exec('helm version') + if (result.code != 0) { + t.fail('helm is not installed') + return + } + + // Add Azure AD Workload Identity Helm Repo + t.is(0, + sh.exec('helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts').code, + 'should be able to add Azure AD workload identity helm repo' + ) + t.is(0, + sh.exec(`helm repo update`).code, + "should be able to update" + ) + + // Install Workload Identity Webhook if not present + t.is(0, + sh.exec(`helm upgrade --install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace ${workloadIdentityNamespace} --create-namespace --set azureTenantID="${AZURE_AD_TENANT_ID}"`).code, + 'should be able to install workload identity webhook' + ) + + let success = false + for (let i = 0; i < 20; i++) { + result = sh.exec( + `kubectl get deployment.apps/azure-wi-webhook-controller-manager -n ${workloadIdentityNamespace} -o jsonpath="{.status.readyReplicas}"` + ) + const parsedPods = parseInt(result.stdout, 10) + if (isNaN(parsedPods) || parsedPods != 2) { + t.log('Workload Identity webhook is not ready. sleeping') + sh.exec('sleep 5s') + } else if (parsedPods == 2) { + t.log('Workload Identity webhook is ready') + success = true + break + } + } + + t.true(success, 'expected workloadd identity deployments to start 2 pods successfully') + + let uri = `https://graph.microsoft.com/beta/applications/${AZURE_AD_OBJECT_ID}/federatedIdentityCredentials` + // Establish federated identity credential + t.is( + 0, + sh.exec(`az rest --method POST --uri ${uri} --body '${federatedCredentialsRequestBody}'`).code, + "should be able to establish federated identity credential" + ) +}) + +test.serial('annotate keda-operator service account for workload identity and redeploy', t => { + t.is( + 0, + sh.exec(`kubectl annotate sa keda-operator -n keda azure.workload.identity/client-id="${AZURE_AD_CLIENT_ID}" --overwrite`).code, + 'should be able to annotate service account' + ) + + t.is(0, + sh.exec(`kubectl annotate sa keda-operator -n keda azure.workload.identity/tenant-id="${AZURE_AD_TENANT_ID}" --overwrite`).code, + 'should be able to annotate service account' + ) + + t.is(0, + sh.exec(`kubectl label sa keda-operator -n keda azure.workload.identity/use=true --overwrite`).code, + 'should be able to annotate service account' + ) + + // Restart keda pods so that the webhook works as expected + t.is(0, + sh.exec(`kubectl rollout restart deployments/keda-operator -n keda`).code, + "should be able to restart keda-operator deployment" + ) + + t.is(0, + sh.exec(`kubectl rollout restart deployments/keda-metrics-apiserver -n keda`).code, + "should be able to restart keda-operator deployment" + ) +}) + test.serial('verifyKeda', t => { let success = false for (let i = 0; i < 20; i++) { @@ -65,15 +167,13 @@ test.serial('verifyKeda', t => { t.true(success, 'expected keda deployments to start 2 pods successfully') }) -test.serial('setup helm', t => { - // check if helm is already installed. - let result = sh.exec('helm version') - if(result.code == 0) { - t.pass('helm is already installed. skipping setup') - return - } - t.is(0, sh.exec(`curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3`).code, 'should be able to download helm script') - t.is(0, sh.exec(`chmod 700 get_helm.sh`).code, 'should be able to change helm script permissions') - t.is(0, sh.exec(`./get_helm.sh`).code, 'should be able to download helm') - t.is(0, sh.exec(`helm version`).code, 'should be able to get helm version') -}) +const federatedCredentialsRequestBody = `{ + "name": "${federatedIdentityCredentialName}", + "issuer": "${SERVICE_ACCOUNT_ISSUER}", + "subject": "system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}", + "description": "KEDA E2E service account federated credential", + "audiences": [ + "api://AzureADTokenExchange" + ] +} +` From 438f5b44d03a0829bc3278655b24381381f0727f Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 29 Apr 2022 14:48:46 +0530 Subject: [PATCH 20/30] Change env variable names. Signed-off-by: Vighnesh Shenoy --- tests/cleanup.test.ts | 2 +- tests/setup.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index 9621f844b2d..69d9778d696 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -8,7 +8,7 @@ test.before('setup shelljs', () => { sh.config.silent = true }) -const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT'] +const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] test.serial('Remove KEDA', t => { let result = sh.exec('(cd .. && make undeploy)') diff --git a/tests/setup.test.ts b/tests/setup.test.ts index 15a32946e18..f25aa43aabb 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -5,10 +5,10 @@ import test from 'ava' const kc = new k8s.KubeConfig() kc.loadFromDefault() -const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_ID'] -const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT'] +const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_APP_ID'] +const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] -const SERVICE_ACCOUNT_ISSUER = process.env['AZURE_OIDC_ISSUER_URL'] +const SERVICE_ACCOUNT_ISSUER = process.env['OIDC_ISSUER_URL'] const SERVICE_ACCOUNT_NAMESPACE = 'keda' const SERVICE_ACCOUNT_NAME = 'keda-operator' const workloadIdentityNamespace = "azure-workload-identity-system" From d31c58ea20fb201ea75aa5eb9e9dcfbc25b05298 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 29 Apr 2022 15:51:07 +0530 Subject: [PATCH 21/30] Make workload identity tests optional. Signed-off-by: Vighnesh Shenoy --- tests/cleanup.test.ts | 10 +++++++--- tests/setup.test.ts | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index 69d9778d696..913a26d0c03 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -3,13 +3,13 @@ import test from 'ava' const workloadIdentityNamespace = "azure-workload-identity-system" const federatedIdentityCredentialName = "keda-e2e-federated-credential" +const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] +const RUN_WORKLOAD_IDENTITY_TESTS = process.env['RUN_WORKLOAD_IDENTITY_TESTS'] test.before('setup shelljs', () => { - sh.config.silent = true + sh.config.silent = false // TODO - Revert after PR workflow runs successfully }) -const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] - test.serial('Remove KEDA', t => { let result = sh.exec('(cd .. && make undeploy)') if (result.code !== 0) { @@ -19,6 +19,10 @@ test.serial('Remove KEDA', t => { }) test.serial('remove azure workload identity kubernetes components', t => { + if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { + t.pass('nothing to clean') + return + } t.is(0, sh.exec(`helm uninstall workload-identity-webhook --namespace ${workloadIdentityNamespace}`).code, diff --git a/tests/setup.test.ts b/tests/setup.test.ts index f25aa43aabb..1d9161a47bb 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -8,6 +8,7 @@ kc.loadFromDefault() const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_APP_ID'] const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] +const RUN_WORKLOAD_IDENTITY_TESTS = process.env['RUN_WORKLOAD_IDENTITY_TESTS'] const SERVICE_ACCOUNT_ISSUER = process.env['OIDC_ISSUER_URL'] const SERVICE_ACCOUNT_NAMESPACE = 'keda' const SERVICE_ACCOUNT_NAME = 'keda-operator' @@ -15,7 +16,7 @@ const workloadIdentityNamespace = "azure-workload-identity-system" const federatedIdentityCredentialName = "keda-e2e-federated-credential" test.before('configure shelljs', () => { - sh.config.silent = true + sh.config.silent = false // TODO - Revert after PR workflow runs successfully }) test.serial('Verify all commands', t => { @@ -64,6 +65,11 @@ test.serial('setup helm', t => { }) test.serial('setup and verify azure workload identity kubernetes components', t => { + if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { + t.pass('nothing to setup') + return + } + // check if helm is already installed. let result = sh.exec('helm version') if (result.code != 0) { From cec4bea6dc864f35b384161e5659b496221e246a Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 29 Apr 2022 16:22:49 +0530 Subject: [PATCH 22/30] Change env variable name. Signed-off-by: Vighnesh Shenoy --- tests/cleanup.test.ts | 2 +- tests/setup.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index 913a26d0c03..e5a2caf5efc 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -4,7 +4,7 @@ import test from 'ava' const workloadIdentityNamespace = "azure-workload-identity-system" const federatedIdentityCredentialName = "keda-e2e-federated-credential" const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] -const RUN_WORKLOAD_IDENTITY_TESTS = process.env['RUN_WORKLOAD_IDENTITY_TESTS'] +const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] test.before('setup shelljs', () => { sh.config.silent = false // TODO - Revert after PR workflow runs successfully diff --git a/tests/setup.test.ts b/tests/setup.test.ts index 1d9161a47bb..f6f00b9f59c 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -8,7 +8,7 @@ kc.loadFromDefault() const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_APP_ID'] const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] -const RUN_WORKLOAD_IDENTITY_TESTS = process.env['RUN_WORKLOAD_IDENTITY_TESTS'] +const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] const SERVICE_ACCOUNT_ISSUER = process.env['OIDC_ISSUER_URL'] const SERVICE_ACCOUNT_NAMESPACE = 'keda' const SERVICE_ACCOUNT_NAME = 'keda-operator' From 516a2ccedbc17927319c9129299a08a4abbcbf06 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 29 Apr 2022 20:12:14 +0530 Subject: [PATCH 23/30] Rework e2e tests. Signed-off-by: Vighnesh Shenoy --- tests/cleanup.test.ts | 12 ++---------- tests/setup.test.ts | 33 +++++++-------------------------- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index e5a2caf5efc..a991ab6bfd5 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -2,12 +2,10 @@ import * as sh from 'shelljs' import test from 'ava' const workloadIdentityNamespace = "azure-workload-identity-system" -const federatedIdentityCredentialName = "keda-e2e-federated-credential" -const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] test.before('setup shelljs', () => { - sh.config.silent = false // TODO - Revert after PR workflow runs successfully + sh.config.silent = true }) test.serial('Remove KEDA', t => { @@ -20,7 +18,7 @@ test.serial('Remove KEDA', t => { test.serial('remove azure workload identity kubernetes components', t => { if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { - t.pass('nothing to clean') + t.pass('skipping as workload identity tests are disabled') return } @@ -30,10 +28,4 @@ test.serial('remove azure workload identity kubernetes components', t => { ) sh.exec(`kubectl delete ns ${workloadIdentityNamespace}`) - - let uri = `https://graph.microsoft.com/beta/applications/${AZURE_AD_OBJECT_ID}/federatedIdentityCredentials/${federatedIdentityCredentialName}` - t.is(0, - sh.exec(`az rest --method DELETE --uri ${uri}`).code, - "should be able to delete federated identity credential" - ) }) diff --git a/tests/setup.test.ts b/tests/setup.test.ts index f6f00b9f59c..ba557e66bf4 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -6,17 +6,12 @@ const kc = new k8s.KubeConfig() kc.loadFromDefault() const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_APP_ID'] -const AZURE_AD_OBJECT_ID = process.env['AZURE_SP_OBJECT_ID'] const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] -const SERVICE_ACCOUNT_ISSUER = process.env['OIDC_ISSUER_URL'] -const SERVICE_ACCOUNT_NAMESPACE = 'keda' -const SERVICE_ACCOUNT_NAME = 'keda-operator' const workloadIdentityNamespace = "azure-workload-identity-system" -const federatedIdentityCredentialName = "keda-e2e-federated-credential" test.before('configure shelljs', () => { - sh.config.silent = false // TODO - Revert after PR workflow runs successfully + sh.config.silent = true }) test.serial('Verify all commands', t => { @@ -66,7 +61,7 @@ test.serial('setup helm', t => { test.serial('setup and verify azure workload identity kubernetes components', t => { if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { - t.pass('nothing to setup') + t.pass('skipping as workload identity tests are disabled') return } @@ -110,17 +105,14 @@ test.serial('setup and verify azure workload identity kubernetes components', t } t.true(success, 'expected workloadd identity deployments to start 2 pods successfully') - - let uri = `https://graph.microsoft.com/beta/applications/${AZURE_AD_OBJECT_ID}/federatedIdentityCredentials` - // Establish federated identity credential - t.is( - 0, - sh.exec(`az rest --method POST --uri ${uri} --body '${federatedCredentialsRequestBody}'`).code, - "should be able to establish federated identity credential" - ) }) test.serial('annotate keda-operator service account for workload identity and redeploy', t => { + if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { + t.pass('skipping as workload identity tests are disabled') + return + } + t.is( 0, sh.exec(`kubectl annotate sa keda-operator -n keda azure.workload.identity/client-id="${AZURE_AD_CLIENT_ID}" --overwrite`).code, @@ -172,14 +164,3 @@ test.serial('verifyKeda', t => { t.true(success, 'expected keda deployments to start 2 pods successfully') }) - -const federatedCredentialsRequestBody = `{ - "name": "${federatedIdentityCredentialName}", - "issuer": "${SERVICE_ACCOUNT_ISSUER}", - "subject": "system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}", - "description": "KEDA E2E service account federated credential", - "audiences": [ - "api://AzureADTokenExchange" - ] -} -` From 9aa37f2aa46d0ec0658ef8bbf8f70a32dfdd72c2 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Tue, 3 May 2022 22:26:24 +0530 Subject: [PATCH 24/30] Change test name. Signed-off-by: Vighnesh Shenoy --- tests/scalers/azure-service-bus-queue-workload-identity.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts index 36aacf55796..f2411ee9332 100644 --- a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts +++ b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts @@ -9,7 +9,7 @@ const connectionString = process.env["AZURE_SERVICE_BUS_CONNECTION_STRING"] const serviceBusNameSpace = connectionString.split("//")[1].split(".")[0] const queueName = "sb-queue" -const testName = "test-azure-service-bus-queue" +const testName = "test-azure-service-bus-queue-workload-identity" const testNamespace = `${testName}-ns` const deploymentName = `${testName}-deployment` const triggerAuthName = `${testName}-trigger-auth` From 54b77224e74a3c323e0aae17a03bed0eb56906db Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 4 May 2022 13:46:30 +0530 Subject: [PATCH 25/30] Use kustomize for setting require labels and annotations on service account. Signed-off-by: Vighnesh Shenoy --- Makefile | 5 +++ config/default/kustomization.yaml | 1 + config/general/kustomization.yaml | 1 - config/service_account/kustomization.yaml | 5 +++ .../service_account.yaml | 0 tests/setup.test.ts | 44 +++---------------- 6 files changed, 16 insertions(+), 40 deletions(-) create mode 100644 config/service_account/kustomization.yaml rename config/{general => service_account}/service_account.yaml (100%) diff --git a/Makefile b/Makefile index d468982c640..11f2d07ce7b 100644 --- a/Makefile +++ b/Makefile @@ -231,6 +231,11 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in $(KUSTOMIZE) edit set image ghcr.io/kedacore/keda-metrics-apiserver=${IMAGE_ADAPTER} # Need this workaround to mitigate a problem with inserting labels into selectors, # until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009 + if [ "$(AZURE_RUN_WORKLOAD_IDENTITY_TESTS)" = true ]; then \ + cd config/service_account && \ + $(KUSTOMIZE) edit add label --force azure.workload.identity/use:true; \ + $(KUSTOMIZE) edit add annotation --force azure.workload.identity/client-id:${AZURE_SP_APP_ID} azure.workload.identity/tenant-id:${AZURE_SP_TENANT}; \ + fi @sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml rm -rf config/default/kustomize-config/metadataLabelTransformer.yaml.out $(KUSTOMIZE) build config/default | kubectl apply -f - diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index cdf3a30e51c..a9a8390e9ea 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -28,3 +28,4 @@ resources: - ../rbac - ../manager - ../metrics-server +- ../service_account diff --git a/config/general/kustomization.yaml b/config/general/kustomization.yaml index 29f4bdc8d83..bf20f4df68b 100644 --- a/config/general/kustomization.yaml +++ b/config/general/kustomization.yaml @@ -1,3 +1,2 @@ resources: - namespace.yaml -- service_account.yaml diff --git a/config/service_account/kustomization.yaml b/config/service_account/kustomization.yaml new file mode 100644 index 00000000000..4256b2be3c7 --- /dev/null +++ b/config/service_account/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- service_account.yaml + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization diff --git a/config/general/service_account.yaml b/config/service_account/service_account.yaml similarity index 100% rename from config/general/service_account.yaml rename to config/service_account/service_account.yaml diff --git a/tests/setup.test.ts b/tests/setup.test.ts index ba557e66bf4..d632658ff26 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -38,14 +38,6 @@ test.serial('Get Kubernetes version', t => { } }) -test.serial('Deploy KEDA', t => { - let result = sh.exec('(cd .. && make deploy)') - if (result.code !== 0) { - t.fail('error deploying keda. ' + result) - } - t.pass('KEDA deployed successfully using make deploy command') -}) - test.serial('setup helm', t => { // check if helm is already installed. let result = sh.exec('helm version') @@ -107,38 +99,12 @@ test.serial('setup and verify azure workload identity kubernetes components', t t.true(success, 'expected workloadd identity deployments to start 2 pods successfully') }) -test.serial('annotate keda-operator service account for workload identity and redeploy', t => { - if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { - t.pass('skipping as workload identity tests are disabled') - return +test.serial('Deploy KEDA', t => { + let result = sh.exec('(cd .. && make deploy)') + if (result.code !== 0) { + t.fail('error deploying keda. ' + result) } - - t.is( - 0, - sh.exec(`kubectl annotate sa keda-operator -n keda azure.workload.identity/client-id="${AZURE_AD_CLIENT_ID}" --overwrite`).code, - 'should be able to annotate service account' - ) - - t.is(0, - sh.exec(`kubectl annotate sa keda-operator -n keda azure.workload.identity/tenant-id="${AZURE_AD_TENANT_ID}" --overwrite`).code, - 'should be able to annotate service account' - ) - - t.is(0, - sh.exec(`kubectl label sa keda-operator -n keda azure.workload.identity/use=true --overwrite`).code, - 'should be able to annotate service account' - ) - - // Restart keda pods so that the webhook works as expected - t.is(0, - sh.exec(`kubectl rollout restart deployments/keda-operator -n keda`).code, - "should be able to restart keda-operator deployment" - ) - - t.is(0, - sh.exec(`kubectl rollout restart deployments/keda-metrics-apiserver -n keda`).code, - "should be able to restart keda-operator deployment" - ) + t.pass('KEDA deployed successfully using make deploy command') }) test.serial('verifyKeda', t => { From 92e0fdb39e4ee406d12a861a210a5ee93b521258 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 4 May 2022 14:08:26 +0530 Subject: [PATCH 26/30] Minor changes. Signed-off-by: Vighnesh Shenoy --- Makefile | 4 ++-- tests/setup.test.ts | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 11f2d07ce7b..323df1f36e0 100644 --- a/Makefile +++ b/Makefile @@ -229,13 +229,13 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in $(KUSTOMIZE) edit set image ghcr.io/kedacore/keda=${IMAGE_CONTROLLER} cd config/metrics-server && \ $(KUSTOMIZE) edit set image ghcr.io/kedacore/keda-metrics-apiserver=${IMAGE_ADAPTER} - # Need this workaround to mitigate a problem with inserting labels into selectors, - # until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009 if [ "$(AZURE_RUN_WORKLOAD_IDENTITY_TESTS)" = true ]; then \ cd config/service_account && \ $(KUSTOMIZE) edit add label --force azure.workload.identity/use:true; \ $(KUSTOMIZE) edit add annotation --force azure.workload.identity/client-id:${AZURE_SP_APP_ID} azure.workload.identity/tenant-id:${AZURE_SP_TENANT}; \ fi + # Need this workaround to mitigate a problem with inserting labels into selectors, + # until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009 @sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml rm -rf config/default/kustomize-config/metadataLabelTransformer.yaml.out $(KUSTOMIZE) build config/default | kubectl apply -f - diff --git a/tests/setup.test.ts b/tests/setup.test.ts index d632658ff26..8cc2a09510b 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -5,13 +5,12 @@ import test from 'ava' const kc = new k8s.KubeConfig() kc.loadFromDefault() -const AZURE_AD_CLIENT_ID = process.env['AZURE_SP_APP_ID'] const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] const workloadIdentityNamespace = "azure-workload-identity-system" test.before('configure shelljs', () => { - sh.config.silent = true + sh.config.silent = false }) test.serial('Verify all commands', t => { @@ -96,7 +95,7 @@ test.serial('setup and verify azure workload identity kubernetes components', t } } - t.true(success, 'expected workloadd identity deployments to start 2 pods successfully') + t.true(success, 'expected workload identity deployments to start 2 pods successfully') }) test.serial('Deploy KEDA', t => { @@ -128,5 +127,14 @@ test.serial('verifyKeda', t => { } } + + t.log( + sh.exec("kubectl get sa keda-operator -n keda").stdout + ) + + t.log( + sh.exec("kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-operator | xargs kubectl -n keda describe pod").stdout + ) + t.true(success, 'expected keda deployments to start 2 pods successfully') }) From 7eb63b79e00b7bd57238954cbfe09b6de4cf6c67 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Thu, 5 May 2022 11:59:31 +0530 Subject: [PATCH 27/30] Add more logs for debugging. Signed-off-by: Vighnesh Shenoy --- tests/setup.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/setup.test.ts b/tests/setup.test.ts index 8cc2a09510b..a1483aa2d6e 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -129,11 +129,11 @@ test.serial('verifyKeda', t => { t.log( - sh.exec("kubectl get sa keda-operator -n keda").stdout + sh.exec("kubectl get sa keda-operator -n keda -o yaml").stdout ) t.log( - sh.exec("kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-operator | xargs kubectl -n keda describe pod").stdout + sh.exec("kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-operator | xargs kubectl -n keda get pod -o yaml").stdout ) t.true(success, 'expected keda deployments to start 2 pods successfully') From da1d3f5f9886c505ae00e47f0b7e3e056bb789b0 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Mon, 9 May 2022 13:41:31 +0530 Subject: [PATCH 28/30] Add sleep for webhook, remove cleanup code. Signed-off-by: Vighnesh Shenoy --- tests/cleanup.test.ts | 14 -------------- tests/setup.test.ts | 7 ++++--- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index a991ab6bfd5..18de6be2fbd 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -15,17 +15,3 @@ test.serial('Remove KEDA', t => { } t.pass('KEDA undeployed successfully using make undeploy command') }) - -test.serial('remove azure workload identity kubernetes components', t => { - if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { - t.pass('skipping as workload identity tests are disabled') - return - } - - t.is(0, - sh.exec(`helm uninstall workload-identity-webhook --namespace ${workloadIdentityNamespace}`).code, - 'should be able to uninstall workload identity webhook' - ) - - sh.exec(`kubectl delete ns ${workloadIdentityNamespace}`) -}) diff --git a/tests/setup.test.ts b/tests/setup.test.ts index a1483aa2d6e..fe8403bbc83 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -10,7 +10,7 @@ const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TES const workloadIdentityNamespace = "azure-workload-identity-system" test.before('configure shelljs', () => { - sh.config.silent = false + sh.config.silent = false // TODO - Remove later }) test.serial('Verify all commands', t => { @@ -69,7 +69,7 @@ test.serial('setup and verify azure workload identity kubernetes components', t 'should be able to add Azure AD workload identity helm repo' ) t.is(0, - sh.exec(`helm repo update`).code, + sh.exec(`helm repo update azure-workload-identity`).code, "should be able to update" ) @@ -91,6 +91,7 @@ test.serial('setup and verify azure workload identity kubernetes components', t } else if (parsedPods == 2) { t.log('Workload Identity webhook is ready') success = true + sh.exec('sleep 120s') // Sleep for some time for webhook to setup properly break } } @@ -127,7 +128,7 @@ test.serial('verifyKeda', t => { } } - + // TODO - Remove later. t.log( sh.exec("kubectl get sa keda-operator -n keda -o yaml").stdout ) From d0268e54d9b806a6070bb8bcf4769195cca96dc0 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Tue, 10 May 2022 01:54:57 +0530 Subject: [PATCH 29/30] Remove debug logs, TODOs, fix changelog. Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 3 +-- tests/cleanup.test.ts | 14 ++++++++++++++ tests/setup.test.ts | 11 +---------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb153529255..a787f5cbea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md ### New -- TODO ([#XXX](https://github.com/kedacore/keda/issue/XXX)) +- **General:** Support for Azure AD Workload Identity as a pod identity provider. ([2487](https://github.com/kedacore/keda/issues/2487)) ### Improvements @@ -76,7 +76,6 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md - **General:** Introduce new GCP Stackdriver Scaler ([#2661](https://github.com/kedacore/keda/issues/2661)) - **General:** Introduce new GCP Storage Scaler ([#2628](https://github.com/kedacore/keda/issues/2628)) - **General:** Provide support for authentication via Azure Key Vault ([#900](https://github.com/kedacore/keda/issues/900)|[#2733](https://github.com/kedacore/keda/issues/2733)) -- **General:** Support for Azure AD Workload Identity as a pod identity provider. ([2487](https://github.com/kedacore/keda/issues/2487)) - **General**: Support for `ValueMetricType` in `ScaledObject` for all scalers except CPU/Memory ([#2030](https://github.com/kedacore/keda/issues/2030)) ### Improvements diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index 18de6be2fbd..a991ab6bfd5 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -15,3 +15,17 @@ test.serial('Remove KEDA', t => { } t.pass('KEDA undeployed successfully using make undeploy command') }) + +test.serial('remove azure workload identity kubernetes components', t => { + if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { + t.pass('skipping as workload identity tests are disabled') + return + } + + t.is(0, + sh.exec(`helm uninstall workload-identity-webhook --namespace ${workloadIdentityNamespace}`).code, + 'should be able to uninstall workload identity webhook' + ) + + sh.exec(`kubectl delete ns ${workloadIdentityNamespace}`) +}) diff --git a/tests/setup.test.ts b/tests/setup.test.ts index fe8403bbc83..5de723e2415 100644 --- a/tests/setup.test.ts +++ b/tests/setup.test.ts @@ -10,7 +10,7 @@ const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TES const workloadIdentityNamespace = "azure-workload-identity-system" test.before('configure shelljs', () => { - sh.config.silent = false // TODO - Remove later + sh.config.silent = true }) test.serial('Verify all commands', t => { @@ -128,14 +128,5 @@ test.serial('verifyKeda', t => { } } - // TODO - Remove later. - t.log( - sh.exec("kubectl get sa keda-operator -n keda -o yaml").stdout - ) - - t.log( - sh.exec("kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-operator | xargs kubectl -n keda get pod -o yaml").stdout - ) - t.true(success, 'expected keda deployments to start 2 pods successfully') }) From 1edd2ed0ecb1f944038500dcd953aba367e39915 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Tue, 10 May 2022 12:27:48 +0530 Subject: [PATCH 30/30] Change queue name in e2e test to avoid conflict. Signed-off-by: Vighnesh Shenoy --- tests/scalers/azure-service-bus-queue-workload-identity.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts index f2411ee9332..656e6ee66ca 100644 --- a/tests/scalers/azure-service-bus-queue-workload-identity.test.ts +++ b/tests/scalers/azure-service-bus-queue-workload-identity.test.ts @@ -7,7 +7,7 @@ const connectionString = process.env["AZURE_SERVICE_BUS_CONNECTION_STRING"] // Format for connection string - // Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=" const serviceBusNameSpace = connectionString.split("//")[1].split(".")[0] -const queueName = "sb-queue" +const queueName = "sb-queue-workload" const testName = "test-azure-service-bus-queue-workload-identity" const testNamespace = `${testName}-ns`