Merge pull request #1386 from shysank/aks_multitenancy

Multitenancy for managed clusters

azure/scope/clients.go

@@ -99,7 +99,7 @@ func (c *AzureClients) setCredentials(subscriptionID, environmentName string) er
 	return err
 }
 
-func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscriptionID, environmentName string, credentialsProvider *AzureCredentialsProvider) error {
+func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscriptionID, environmentName string, credentialsProvider CredentialsProvider) error {
 	if credentialsProvider == nil {
 		return fmt.Errorf("credentials provider cannot have an empty value")
 	}

azure/scope/cluster.go

@@ -67,7 +67,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc
 			return nil, errors.Wrap(err, "failed to configure azure settings and credentials from environment")
 		}
 	} else {
-		credentailsProvider, err := NewAzureCredentialsProvider(ctx, params.Client, params.AzureCluster)
+		credentailsProvider, err := NewAzureClusterCredentialsProvider(ctx, params.Client, params.AzureCluster)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to init credentials provider")
 		}

azure/scope/identity.go

@@ -25,6 +25,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/pkg/errors"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
+	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
 	"sigs.k8s.io/cluster-api-provider-azure/util/identity"
 	"sigs.k8s.io/cluster-api-provider-azure/util/system"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
@@ -37,17 +38,36 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// AzureCredentialsProvider provides credentials for an AzureCluster.
+// CredentialsProvider defines the behavior for azure identity based credential providers.
+type CredentialsProvider interface {
+	GetAuthorizer(ctx context.Context, resourceManagerEndpoint string) (autorest.Authorizer, error)
+}
+
+// AzureCredentialsProvider represents a credential provider with azure cluster identity.
 type AzureCredentialsProvider struct {
-	Client       client.Client
+	Client   client.Client
+	Identity *infrav1.AzureClusterIdentity
+}
+
+// AzureClusterCredentialsProvider wraps AzureCredentialsProvider with AzureCluster.
+type AzureClusterCredentialsProvider struct {
+	AzureCredentialsProvider
 	AzureCluster *infrav1.AzureCluster
-	Identity     *infrav1.AzureClusterIdentity
 }
 
-// NewAzureCredentialsProvider creates a new AzureCredentialsProvider from the supplied inputs.
-func NewAzureCredentialsProvider(ctx context.Context, kubeClient client.Client, azureCluster *infrav1.AzureCluster) (*AzureCredentialsProvider, error) {
+// ManagedControlPlaneCredentialsProvider wraps AzureCredentialsProvider with AzureManagedControlPlane.
+type ManagedControlPlaneCredentialsProvider struct {
+	AzureCredentialsProvider
+	AzureManagedControlPlane *infrav1exp.AzureManagedControlPlane
+}
+
+var _ CredentialsProvider = (*AzureClusterCredentialsProvider)(nil)
+var _ CredentialsProvider = (*ManagedControlPlaneCredentialsProvider)(nil)
+
+// NewAzureClusterCredentialsProvider creates a new AzureClusterCredentialsProvider from the supplied inputs.
+func NewAzureClusterCredentialsProvider(ctx context.Context, kubeClient client.Client, azureCluster *infrav1.AzureCluster) (*AzureClusterCredentialsProvider, error) {
 	if azureCluster.Spec.IdentityRef == nil {
-		return nil, errors.New("failed to generate new AzureCredentialsProvider from empty identityName")
+		return nil, errors.New("failed to generate new AzureClusterCredentialsProvider from empty identityName")
 	}
 
 	ref := azureCluster.Spec.IdentityRef
@@ -66,15 +86,58 @@ func NewAzureCredentialsProvider(ctx context.Context, kubeClient client.Client,
 		return nil, errors.New("AzureClusterIdentity is not of type Service Principal")
 	}
 
-	return &AzureCredentialsProvider{
-		Client:       kubeClient,
-		AzureCluster: azureCluster,
-		Identity:     identity,
+	return &AzureClusterCredentialsProvider{
+		AzureCredentialsProvider{
+			Client:   kubeClient,
+			Identity: identity,
+		},
+		azureCluster,
 	}, nil
 }
 
-// GetAuthorizer returns an Azure authorizer based on the provided Azure identity.
-func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string) (autorest.Authorizer, error) {
+// GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureCluster metadata.
+func (p *AzureClusterCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string) (autorest.Authorizer, error) {
+	return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, p.AzureCluster.ObjectMeta)
+}
+
+// NewManagedControlPlaneCredentialsProvider creates a new ManagedControlPlaneCredentialsProvider from the supplied inputs.
+func NewManagedControlPlaneCredentialsProvider(ctx context.Context, kubeClient client.Client, managedControlPlane *infrav1exp.AzureManagedControlPlane) (*ManagedControlPlaneCredentialsProvider, error) {
+	if managedControlPlane.Spec.IdentityRef == nil {
+		return nil, errors.New("failed to generate new ManagedControlPlaneCredentialsProvider from empty identityName")
+	}
+
+	ref := managedControlPlane.Spec.IdentityRef
+	// if the namespace isn't specified then assume it's in the same namespace as the AzureManagedControlPlane
+	namespace := ref.Namespace
+	if namespace == "" {
+		namespace = managedControlPlane.Namespace
+	}
+	identity := &infrav1.AzureClusterIdentity{}
+	key := client.ObjectKey{Name: ref.Name, Namespace: namespace}
+	if err := kubeClient.Get(ctx, key, identity); err != nil {
+		return nil, errors.Errorf("failed to retrieve AzureClusterIdentity external object %q/%q: %v", key.Namespace, key.Name, err)
+	}
+
+	if identity.Spec.Type != infrav1.ServicePrincipal {
+		return nil, errors.New("AzureClusterIdentity is not of type Service Principal")
+	}
+
+	return &ManagedControlPlaneCredentialsProvider{
+		AzureCredentialsProvider{
+			Client:   kubeClient,
+			Identity: identity,
+		},
+		managedControlPlane,
+	}, nil
+}
+
+// GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureManagedControlPlane metadata.
+func (p *ManagedControlPlaneCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string) (autorest.Authorizer, error) {
+	return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, p.AzureManagedControlPlane.ObjectMeta)
+}
+
+// GetAuthorizer returns an Azure authorizer based on the provided azure identity and cluster metadata.
+func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string, clusterMeta metav1.ObjectMeta) (autorest.Authorizer, error) {
 	azureIdentityType, err := getAzureIdentityType(p.Identity)
 	if err != nil {
 		return nil, err
@@ -85,17 +148,17 @@ func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceMa
 			APIVersion: "aadpodidentity.k8s.io/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      identity.GetAzureIdentityName(p.AzureCluster.Name, p.AzureCluster.Namespace, p.Identity.Name),
+			Name:      identity.GetAzureIdentityName(clusterMeta.Name, clusterMeta.Namespace, p.Identity.Name),
 			Namespace: system.GetManagerNamespace(),
 			Annotations: map[string]string{
 				aadpodv1.BehaviorKey: "namespaced",
 			},
 			Labels: map[string]string{
-				clusterv1.ClusterLabelName:         p.AzureCluster.Name,
-				infrav1.ClusterLabelNamespace:      p.AzureCluster.Namespace,
+				clusterv1.ClusterLabelName:         clusterMeta.Name,
+				infrav1.ClusterLabelNamespace:      clusterMeta.Namespace,
 				clusterctl.ClusterctlMoveLabelName: "true",
 			},
-			OwnerReferences: p.AzureCluster.OwnerReferences,
+			OwnerReferences: clusterMeta.OwnerReferences,
 		},
 		Spec: aadpodv1.AzureIdentitySpec{
 			Type:           azureIdentityType,
@@ -119,11 +182,11 @@ func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceMa
 			Name:      fmt.Sprintf("%s-binding", copiedIdentity.Name),
 			Namespace: copiedIdentity.Namespace,
 			Labels: map[string]string{
-				clusterv1.ClusterLabelName:         p.AzureCluster.Name,
-				infrav1.ClusterLabelNamespace:      p.AzureCluster.Namespace,
+				clusterv1.ClusterLabelName:         clusterMeta.Name,
+				infrav1.ClusterLabelNamespace:      clusterMeta.Namespace,
 				clusterctl.ClusterctlMoveLabelName: "true",
 			},
-			OwnerReferences: p.AzureCluster.OwnerReferences,
+			OwnerReferences: clusterMeta.OwnerReferences,
 		},
 		Spec: aadpodv1.AzureIdentityBindingSpec{
 			AzureIdentity: copiedIdentity.Name,
@@ -169,7 +232,7 @@ func IsClusterNamespaceAllowed(ctx context.Context, k8sClient client.Client, all
 		return false
 	}
 
-	// empty value matches with all namespaces.
+	// empty value matches with all namespaces
 	if reflect.DeepEqual(*allowedNamespaces, infrav1.AllowedNamespaces{}) {
 		return true
 	}

azure/scope/managedcontrolplane.go

@@ -48,7 +48,7 @@ type ManagedControlPlaneScopeParams struct {
 
 // NewManagedControlPlaneScope creates a new Scope from the supplied parameters.
 // This is meant to be called for each reconcile iteration.
-func NewManagedControlPlaneScope(params ManagedControlPlaneScopeParams) (*ManagedControlPlaneScope, error) {
+func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlaneScopeParams) (*ManagedControlPlaneScope, error) {
 	if params.Cluster == nil {
 		return nil, errors.New("failed to generate new scope from nil Cluster")
 	}
@@ -61,8 +61,19 @@ func NewManagedControlPlaneScope(params ManagedControlPlaneScopeParams) (*Manage
 		params.Logger = klogr.New()
 	}
 
-	if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, ""); err != nil {
-		return nil, errors.Wrap(err, "failed to create Azure session")
+	if params.ControlPlane.Spec.IdentityRef == nil {
+		if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, ""); err != nil {
+			return nil, errors.Wrap(err, "failed to create Azure session")
+		}
+	} else {
+		credentialsProvider, err := NewManagedControlPlaneCredentialsProvider(ctx, params.Client, params.ControlPlane)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to init credentials provider")
+		}
+
+		if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, "", credentialsProvider); err != nil {
+			return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity")
+		}
 	}
 
 	helper, err := patch.NewHelper(params.PatchTarget, params.Client)

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml

@@ -192,6 +192,31 @@ spec:
               dnsServiceIP:
                 description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr.
                 type: string
+              identityRef:
+                description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
+                  namespace:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
+                  resourceVersion:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
+                type: object
               loadBalancerSKU:
                 description: LoadBalancerSKU is the SKU of the loadBalancer to be provisioned.
                 enum:

controllers/azureidentity_controller.go

@@ -18,8 +18,12 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	infraexpv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
+	"sigs.k8s.io/cluster-api-provider-azure/feature"
+
 	aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
@@ -65,6 +69,17 @@ func (r *AzureIdentityReconciler) SetupWithManager(ctx context.Context, mgr ctrl
 		return errors.Wrap(err, "error creating controller")
 	}
 
+	// Add a watch on infraexpv1.AzureManagedControlPlane if aks is enabled.
+	if feature.Gates.Enabled(feature.AKS) {
+		if err = c.Watch(
+			&source.Kind{Type: &infraexpv1.AzureManagedControlPlane{}},
+			&handler.EnqueueRequestForObject{},
+			predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue),
+		); err != nil {
+			return errors.Wrap(err, "failed adding a watch for ready clusters")
+		}
+	}
+
 	// Add a watch on clusterv1.Cluster object for unpause notifications.
 	if err = c.Watch(
 		&source.Kind{Type: &clusterv1.Cluster{}},
@@ -96,20 +111,38 @@ func (r *AzureIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		))
 	defer span.End()
 
+	// identityOwner is the resource that created the identity. This could be either an AzureCluster or AzureManagedControlPlane (if AKS is enabled).
+	// check for AzureManagedControlPlane first and if it is not found, check for AzureManagedControlPlane.
+	var identityOwner interface{}
+
 	// Fetch the AzureCluster instance
 	azureCluster := &infrav1.AzureCluster{}
+	identityOwner = azureCluster
 	err := r.Get(ctx, req.NamespacedName, azureCluster)
-	if err != nil {
-		if apierrors.IsNotFound(err) {
+	if err != nil && apierrors.IsNotFound(err) {
+		if feature.Gates.Enabled(feature.AKS) {
+			// Fetch the AzureManagedControlPlane instance
+			azureManagedControlPlane := &infraexpv1.AzureManagedControlPlane{}
+			identityOwner = azureManagedControlPlane
+			err = r.Get(ctx, req.NamespacedName, azureManagedControlPlane)
+			if err != nil && apierrors.IsNotFound(err) {
+				r.Recorder.Eventf(azureCluster, corev1.EventTypeNormal, "AzureClusterObjectNotFound",
+					fmt.Sprintf("AzureCluster object %s/%s not found", req.Namespace, req.Name))
+				r.Recorder.Eventf(azureManagedControlPlane, corev1.EventTypeNormal, "AzureManagedControlPlaneObjectNotFound",
+					fmt.Sprintf("AzureManagedControlPlane object %s/%s not found", req.Namespace, req.Name))
+				log.Info("object was not found")
+				return reconcile.Result{}, nil
+			}
+		} else {
 			r.Recorder.Eventf(azureCluster, corev1.EventTypeNormal, "AzureClusterObjectNotFound", err.Error())
 			log.Info("object was not found")
 			return reconcile.Result{}, nil
 		}
+	}
+	if err != nil {
 		return reconcile.Result{}, err
 	}
 
-	log = log.WithValues("azurecluster", azureCluster.Name)
-
 	// get all the bindings
 	var bindings aadpodv1.AzureIdentityBindingList
 	if err := r.List(ctx, &bindings, client.InNamespace(system.GetManagerNamespace())); err != nil {
@@ -125,16 +158,36 @@ func (r *AzureIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		clusterNamespace := binding.ObjectMeta.Labels[infrav1.ClusterLabelNamespace]
 
 		key := client.ObjectKey{Name: clusterName, Namespace: clusterNamespace}
-		azCluster := &infrav1.AzureCluster{}
-		if err := r.Get(ctx, key, azCluster); err != nil {
-			if apierrors.IsNotFound(err) {
-				bindingsToDelete = append(bindingsToDelete, b)
-				continue
-			} else {
-				return ctrl.Result{}, errors.Wrap(err, "failed to get AzureCluster")
+		var expectedIdentityName string
+
+		// only delete bindings when the identity owner type is not found.
+		// we should not delete an identity when azureCluster is not found because it could have been created by AzureManagedControlPlane.
+		switch identityOwner.(type) {
+		case infrav1.AzureCluster:
+			azCluster := &infrav1.AzureCluster{}
+			if err := r.Get(ctx, key, azCluster); err != nil {
+				if apierrors.IsNotFound(err) {
+					bindingsToDelete = append(bindingsToDelete, b)
+					continue
+				} else {
+					return ctrl.Result{}, errors.Wrap(err, "failed to get AzureCluster")
+				}
+			}
+			expectedIdentityName = identity.GetAzureIdentityName(azCluster.Name, azCluster.Namespace, azCluster.Spec.IdentityRef.Name)
+		case infraexpv1.AzureManagedControlPlane:
+			azManagedControlPlane := &infraexpv1.AzureManagedControlPlane{}
+			if err := r.Get(ctx, key, azManagedControlPlane); err != nil {
+				if apierrors.IsNotFound(err) {
+					bindingsToDelete = append(bindingsToDelete, b)
+					continue
+				} else {
+					return ctrl.Result{}, errors.Wrap(err, "failed to get AzureManagedControlPlane")
+				}
 			}
+			expectedIdentityName = identity.GetAzureIdentityName(azManagedControlPlane.Name, azManagedControlPlane.Namespace,
+				azManagedControlPlane.Spec.IdentityRef.Name)
 		}
-		expectedIdentityName := identity.GetAzureIdentityName(azCluster.Name, azCluster.Namespace, azCluster.Spec.IdentityRef.Name)
+
 		if binding.Spec.AzureIdentity != expectedIdentityName {
 			bindingsToDelete = append(bindingsToDelete, b)
 		}
@@ -145,7 +198,7 @@ func (r *AzureIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		binding := bindingToDelete
 		identityName := binding.Spec.AzureIdentity
 		if err := r.Client.Delete(ctx, &binding); err != nil {
-			r.Recorder.Eventf(azureCluster, corev1.EventTypeWarning, "Error reconciling AzureIdentity for AzureCluster", err.Error())
+			r.Recorder.Eventf(azureCluster, corev1.EventTypeWarning, "Error reconciling AzureIdentity", err.Error())
 			log.Error(err, "failed to delete AzureIdentityBinding")
 			return ctrl.Result{}, err
 		}
@@ -155,7 +208,7 @@ func (r *AzureIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 			return ctrl.Result{}, err
 		}
 		if err := r.Client.Delete(ctx, azureIdentity); err != nil {
-			r.Recorder.Eventf(azureCluster, corev1.EventTypeWarning, "Error reconciling AzureIdentity for AzureCluster", err.Error())
+			r.Recorder.Eventf(azureCluster, corev1.EventTypeWarning, "Error reconciling AzureIdentity", err.Error())
 			log.Error(err, "failed to delete AzureIdentity")
 			return ctrl.Result{}, err
 		}