From ad3eb5ca4760cc3e6ebb867be365e55e3bbf14cb Mon Sep 17 00:00:00 2001 From: Soule BA Date: Wed, 24 Aug 2022 09:29:19 +0200 Subject: [PATCH] Enable contextual login for helm OCI If implemented, this pr will enable user to use the auto login feature in order to automatically login to their provider of choice's container registry (i.e. aws, gcr, acr). Signed-off-by: Soule BA --- api/v1beta2/helmrepository_types.go | 12 +- ...ce.toolkit.fluxcd.io_helmrepositories.yaml | 16 ++- controllers/helmchart_controller.go | 34 +++++ controllers/helmchart_controller_test.go | 7 +- controllers/helmrepository_controller_oci.go | 59 ++++++++ .../helmrepository_controller_oci_test.go | 3 +- docs/api/source.md | 36 ++++- docs/spec/v1beta2/helmrepositories.md | 128 ++++++++++++++++++ docs/spec/v1beta2/ocirepositories.md | 2 +- internal/helm/registry/auth.go | 23 ++++ internal/helm/registry/auth_test.go | 52 +++++++ 11 files changed, 363 insertions(+), 9 deletions(-) diff --git a/api/v1beta2/helmrepository_types.go b/api/v1beta2/helmrepository_types.go index 87c0b16b8..d9d72b0b9 100644 --- a/api/v1beta2/helmrepository_types.go +++ b/api/v1beta2/helmrepository_types.go @@ -68,7 +68,9 @@ type HelmRepositorySpec struct { // +required Interval metav1.Duration `json:"interval"` - // Timeout of the index fetch operation, defaults to 60s. + // Timeout is used for the index fetch operation for an HTTPS helm repository, + // and for remote OCI Repository operations like pulling for an OCI helm repository. + // Its default value is 60s. // +kubebuilder:default:="60s" // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` @@ -89,6 +91,14 @@ type HelmRepositorySpec struct { // +kubebuilder:validation:Enum=default;oci // +optional Type string `json:"type,omitempty"` + + // Provider used for authentication, can be 'aws', 'azure', 'gcp' or 'generic'. + // This field is optional, and only taken into account if the .spec.type field is set to 'oci'. + // When not specified, defaults to 'generic'. + // +kubebuilder:validation:Enum=generic;aws;azure;gcp + // +kubebuilder:default:=generic + // +optional + Provider string `json:"provider,omitempty"` } // HelmRepositoryStatus records the observed state of the HelmRepository. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml index c19552fdd..3aba3cf9a 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml @@ -310,6 +310,18 @@ spec: be done with caution, as it can potentially result in credentials getting stolen in a MITM-attack. type: boolean + provider: + default: generic + description: Provider used for authentication, can be 'aws', 'azure', + 'gcp' or 'generic'. This field is optional, and only taken into + account if the .spec.type field is set to 'oci'. When not specified, + defaults to 'generic'. + enum: + - generic + - aws + - azure + - gcp + type: string secretRef: description: SecretRef specifies the Secret containing authentication credentials for the HelmRepository. For HTTP/S basic auth the secret @@ -328,7 +340,9 @@ spec: type: boolean timeout: default: 60s - description: Timeout of the index fetch operation, defaults to 60s. + description: Timeout is used for the index fetch operation for an + HTTPS helm repository, and for remote OCI Repository operations + like pulling for an OCI helm repository. Its default value is 60s. type: string type: description: Type of the HelmRepository. When this field is set to "oci", diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 032f678bb..fd1714724 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/oci" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" "github.com/fluxcd/pkg/runtime/events" @@ -463,6 +464,9 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj * tlsConfig *tls.Config loginOpts []helmreg.LoginOption ) + // Used to login with the repository declared provider + ctxTimeout, cancel := context.WithTimeout(ctx, repo.Spec.Timeout.Duration) + defer cancel() normalizedURL := repository.NormalizeURL(repo.Spec.URL) // Construct the Getter options from the HelmRepository data @@ -521,6 +525,21 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj * loginOpts = append([]helmreg.LoginOption{}, loginOpt) } + if repo.Spec.Provider != sourcev1.GenericOCIProvider && repo.Spec.Type == sourcev1.HelmRepositoryTypeOCI { + auth, authErr := oidcAuth(ctxTimeout, repo) + if authErr != nil && !errors.Is(authErr, oci.ErrUnconfiguredProvider) { + e := &serror.Event{ + Err: fmt.Errorf("failed to get credential from %s: %w", repo.Spec.Provider, authErr), + Reason: sourcev1.AuthenticationFailedReason, + } + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + if auth != nil { + loginOpts = append([]helmreg.LoginOption{}, auth) + } + } + // Initialize the chart repository var chartRepo repository.Downloader switch repo.Spec.Type { @@ -947,6 +966,11 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont }, } } + + // Used to login with the repository declared provider + ctxTimeout, cancel := context.WithTimeout(ctx, repo.Spec.Timeout.Duration) + defer cancel() + clientOpts := []helmgetter.Option{ helmgetter.WithURL(normalizedURL), helmgetter.WithTimeout(repo.Spec.Timeout.Duration), @@ -976,6 +1000,16 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont loginOpts = append([]helmreg.LoginOption{}, loginOpt) } + if repo.Spec.Provider != sourcev1.GenericOCIProvider && repo.Spec.Type == sourcev1.HelmRepositoryTypeOCI { + auth, authErr := oidcAuth(ctxTimeout, repo) + if authErr != nil && !errors.Is(authErr, oci.ErrUnconfiguredProvider) { + return nil, fmt.Errorf("failed to get credential from %s: %w", repo.Spec.Provider, authErr) + } + if auth != nil { + loginOpts = append([]helmreg.LoginOption{}, auth) + } + } + var chartRepo repository.Downloader if helmreg.IsOCI(normalizedURL) { registryClient, credentialsFile, err := r.RegistryClientGenerator(loginOpts != nil) diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index 26e771c5a..e9c3920d2 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -1085,9 +1085,10 @@ func TestHelmChartReconciler_buildFromOCIHelmRepository(t *testing.T) { GenerateName: "helmrepository-", }, Spec: sourcev1.HelmRepositorySpec{ - URL: fmt.Sprintf("oci://%s/testrepo", testRegistryServer.registryHost), - Timeout: &metav1.Duration{Duration: timeout}, - Type: sourcev1.HelmRepositoryTypeOCI, + URL: fmt.Sprintf("oci://%s/testrepo", testRegistryServer.registryHost), + Timeout: &metav1.Duration{Duration: timeout}, + Provider: sourcev1.GenericOCIProvider, + Type: sourcev1.HelmRepositoryTypeOCI, }, } obj := &sourcev1.HelmChart{ diff --git a/controllers/helmrepository_controller_oci.go b/controllers/helmrepository_controller_oci.go index a7d812fa0..75f1ccbf0 100644 --- a/controllers/helmrepository_controller_oci.go +++ b/controllers/helmrepository_controller_oci.go @@ -22,6 +22,7 @@ import ( "fmt" "net/url" "os" + "strings" "time" helmgetter "helm.sh/helm/v3/pkg/getter" @@ -41,10 +42,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/oci" + "github.com/fluxcd/pkg/oci/auth/login" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" + "github.com/google/go-containerregistry/pkg/name" "github.com/fluxcd/source-controller/api/v1beta2" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" @@ -204,6 +208,9 @@ func (r *HelmRepositoryOCIReconciler) Reconcile(ctx context.Context, req ctrl.Re // block at the very end to summarize the conditions to be in a consistent // state. func (r *HelmRepositoryOCIReconciler) reconcile(ctx context.Context, obj *v1beta2.HelmRepository) (result ctrl.Result, retErr error) { + ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration) + defer cancel() + oldObj := obj.DeepCopy() defer func() { @@ -296,6 +303,19 @@ func (r *HelmRepositoryOCIReconciler) reconcile(ctx context.Context, obj *v1beta } } + if obj.Spec.Provider != sourcev1.GenericOCIProvider && obj.Spec.Type == sourcev1.HelmRepositoryTypeOCI { + auth, authErr := oidcAuth(ctxTimeout, obj) + if authErr != nil && !errors.Is(authErr, oci.ErrUnconfiguredProvider) { + e := fmt.Errorf("failed to get credential from %s: %w", obj.Spec.Provider, authErr) + conditions.MarkFalse(obj, meta.ReadyCondition, sourcev1.AuthenticationFailedReason, e.Error()) + result, retErr = ctrl.Result{}, e + return + } + if auth != nil { + loginOpts = append(loginOpts, auth) + } + } + // Create registry client and login if needed. registryClient, file, err := r.RegistryClientGenerator(loginOpts != nil) if err != nil { @@ -366,3 +386,42 @@ func (r *HelmRepositoryOCIReconciler) eventLogf(ctx context.Context, obj runtime } r.Eventf(obj, eventType, reason, msg) } + +// oidcAuth generates the OIDC credential authenticator based on the specified cloud provider. +func oidcAuth(ctx context.Context, obj *sourcev1.HelmRepository) (helmreg.LoginOption, error) { + url := strings.TrimPrefix(obj.Spec.URL, sourcev1.OCIRepositoryPrefix) + ref, err := name.ParseReference(url) + if err != nil { + return nil, fmt.Errorf("failed to parse URL '%s': %w", obj.Spec.URL, err) + } + + loginOpt, err := loginWithManager(ctx, obj.Spec.Provider, url, ref) + if err != nil { + return nil, fmt.Errorf("failed to login to registry '%s': %w", obj.Spec.URL, err) + } + + return loginOpt, nil +} + +func loginWithManager(ctx context.Context, provider, url string, ref name.Reference) (helmreg.LoginOption, error) { + opts := login.ProviderOptions{} + switch provider { + case sourcev1.AmazonOCIProvider: + opts.AwsAutoLogin = true + case sourcev1.AzureOCIProvider: + opts.AzureAutoLogin = true + case sourcev1.GoogleOCIProvider: + opts.GcpAutoLogin = true + } + + auth, err := login.NewManager().Login(ctx, url, ref, opts) + if err != nil { + return nil, err + } + + if auth == nil { + return nil, nil + } + + return registry.OIDCAdaptHelper(auth) +} diff --git a/controllers/helmrepository_controller_oci_test.go b/controllers/helmrepository_controller_oci_test.go index 62d49ec29..ec75a67ef 100644 --- a/controllers/helmrepository_controller_oci_test.go +++ b/controllers/helmrepository_controller_oci_test.go @@ -94,7 +94,8 @@ func TestHelmRepositoryOCIReconciler_Reconcile(t *testing.T) { SecretRef: &meta.LocalObjectReference{ Name: secret.Name, }, - Type: sourcev1.HelmRepositoryTypeOCI, + Provider: sourcev1.GenericOCIProvider, + Type: sourcev1.HelmRepositoryTypeOCI, }, } g.Expect(testEnv.Create(ctx, obj)).To(Succeed()) diff --git a/docs/api/source.md b/docs/api/source.md index ec0b1daf7..47368ddc4 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -818,7 +818,9 @@ Kubernetes meta/v1.Duration (Optional) -

Timeout of the index fetch operation, defaults to 60s.

+

Timeout is used for the index fetch operation for an HTTPS helm repository, +and for remote OCI Repository operations like pulling for an OCI helm repository. +Its default value is 60s.

@@ -863,6 +865,20 @@ string When this field is set to “oci”, the URL field value must be prefixed with “oci://”.

+ + +provider
+ +string + + + +(Optional) +

Provider used for authentication, can be ‘aws’, ‘azure’, ‘gcp’ or ‘generic’. +This field is optional, and only taken into account if the .spec.type field is set to ‘oci’. +When not specified, defaults to ‘generic’.

+ + @@ -2347,7 +2363,9 @@ Kubernetes meta/v1.Duration (Optional) -

Timeout of the index fetch operation, defaults to 60s.

+

Timeout is used for the index fetch operation for an HTTPS helm repository, +and for remote OCI Repository operations like pulling for an OCI helm repository. +Its default value is 60s.

@@ -2392,6 +2410,20 @@ string When this field is set to “oci”, the URL field value must be prefixed with “oci://”.

+ + +provider
+ +string + + + +(Optional) +

Provider used for authentication, can be ‘aws’, ‘azure’, ‘gcp’ or ‘generic’. +This field is optional, and only taken into account if the .spec.type field is set to ‘oci’. +When not specified, defaults to ‘generic’.

+ + diff --git a/docs/spec/v1beta2/helmrepositories.md b/docs/spec/v1beta2/helmrepositories.md index a77902882..29a3832c1 100644 --- a/docs/spec/v1beta2/helmrepositories.md +++ b/docs/spec/v1beta2/helmrepositories.md @@ -162,6 +162,134 @@ A HelmRepository also needs a Possible values are `default` for a Helm HTTP/S repository, or `oci` for an OCI Helm repository. + +### Provider + +`.spec.provider` is an optional field that allows specifying an OIDC provider used +for authentication purposes. + +Supported options are: +- `generic` +- `aws` +- `azure` +- `gcp` + +The `generic` provider can be used for public repositories or when static credentials +are used for authentication. If you do not specify `.spec.provider`, it defaults +to `generic`. + +**Note**: The provider field is supported only for Helm OCI repositories. The `spec.type` +field must be set to `oci`. + +#### AWS + +The `aws` provider can be used to authenticate automatically using the EKS worker +node IAM role or IAM Role for Service Accounts (IRSA), and by extension gain access +to ECR. + +When the worker node IAM role has access to ECR, source-controller running on it +will also have access to ECR. + +When using IRSA to enable access to ECR, add the following patch to your bootstrap +repository, in the `flux-system/kustomization.yaml` file: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gotk-components.yaml + - gotk-sync.yaml +patches: + - patch: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: source-controller + annotations: + eks.amazonaws.com/role-arn: + target: + kind: ServiceAccount + name: source-controller +``` + +Note that you can attach the AWS managed policy `arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly` +to the IAM role when using IRSA. + +#### Azure + +The `azure` provider can be used to authenticate automatically using kubelet managed +identity or Azure Active Directory pod-managed identity (aad-pod-identity), and +by extension gain access to ACR. + +When the kubelet managed identity has access to ACR, source-controller running on +it will also have access to ACR. + +When using aad-pod-identity to enable access to ACR, add the following patch to +your bootstrap repository, in the `flux-system/kustomization.yaml` file: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gotk-components.yaml + - gotk-sync.yaml +patches: + - patch: | + - op: add + path: /spec/template/metadata/labels/aadpodidbinding + value: + target: + kind: Deployment + name: source-controller +``` + +When using pod-managed identity on an AKS cluster, AAD Pod Identity has to be used +to give the `source-controller` pod access to the ACR. To do this, you have to install +`aad-pod-identity` on your cluster, create a managed identity that has access to the +container registry (this can also be the Kubelet identity if it has `AcrPull` role +assignment on the ACR), create an `AzureIdentity` and `AzureIdentityBinding` that describe +the managed identity and then label the `source-controller` pods with the name of the +AzureIdentity as shown in the patch above. Please take a look at [this guide](https://azure.github.io/aad-pod-identity/docs/) +or [this one](https://docs.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity) +if you want to use AKS pod-managed identities add-on that is in preview. + +#### GCP + +The `gcp` provider can be used to authenticate automatically using OAuth scopes or +Workload Identity, and by extension gain access to GCR or Artifact Registry. + +When the GKE nodes have the appropriate OAuth scope for accessing GCR and Artifact Registry, +source-controller running on it will also have access to them. + +When using Workload Identity to enable access to GCR or Artifact Registry, add the +following patch to your bootstrap repository, in the `flux-system/kustomization.yaml` +file: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gotk-components.yaml + - gotk-sync.yaml +patches: + - patch: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: source-controller + annotations: + iam.gke.io/gcp-service-account: + target: + kind: ServiceAccount + name: source-controller +``` + +The Artifact Registry service uses the permission `artifactregistry.repositories.downloadArtifacts` +that is located under the Artifact Registry Reader role. If you are using Google Container Registry service, +the needed permission is instead `storage.objects.list` which can be bound as part +of the Container Registry Service Agent role. Take a look at [this guide](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) +for more information about setting up GKE Workload Identity. + ### Interval `.spec.interval` is a required field that specifies the interval which the diff --git a/docs/spec/v1beta2/ocirepositories.md b/docs/spec/v1beta2/ocirepositories.md index 6bb67650b..095adeee7 100644 --- a/docs/spec/v1beta2/ocirepositories.md +++ b/docs/spec/v1beta2/ocirepositories.md @@ -161,7 +161,7 @@ and by extension gain access to ACR. When the kubelet managed identity has access to ACR, source-controller running on it will also have access to ACR. -When using aad-pod-identity to enable access to ECR, add the following patch to +When using aad-pod-identity to enable access to ACR, add the following patch to your bootstrap repository, in the `flux-system/kustomization.yaml` file: ```yaml diff --git a/internal/helm/registry/auth.go b/internal/helm/registry/auth.go index 75667f1d5..e45d05172 100644 --- a/internal/helm/registry/auth.go +++ b/internal/helm/registry/auth.go @@ -23,6 +23,7 @@ import ( "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/credentials" + "github.com/google/go-containerregistry/pkg/authn" "helm.sh/helm/v3/pkg/registry" corev1 "k8s.io/api/core/v1" ) @@ -68,3 +69,25 @@ func LoginOptionFromSecret(registryURL string, secret corev1.Secret) (registry.L } return registry.LoginOptBasicAuth(username, password), nil } + +// OIDCAdaptHelper returns an ORAS credentials callback configured with the authorization data +// from the given authn authenticator. This allows for example to make use of credential helpers from +// cloud providers. +// Ref: https://github.com/google/go-containerregistry/tree/main/pkg/authn +func OIDCAdaptHelper(authenticator authn.Authenticator) (registry.LoginOption, error) { + authConfig, err := authenticator.Authorization() + if err != nil { + return nil, fmt.Errorf("unable to get authentication data from OIDC: %w", err) + } + + username := authConfig.Username + password := authConfig.Password + + switch { + case username == "" && password == "": + return nil, nil + case username == "" || password == "": + return nil, fmt.Errorf("invalid auth data: required fields 'username' and 'password'") + } + return registry.LoginOptBasicAuth(username, password), nil +} diff --git a/internal/helm/registry/auth_test.go b/internal/helm/registry/auth_test.go index 921ecbf14..58dbd04bf 100644 --- a/internal/helm/registry/auth_test.go +++ b/internal/helm/registry/auth_test.go @@ -19,6 +19,7 @@ package registry import ( "testing" + "github.com/google/go-containerregistry/pkg/authn" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" ) @@ -129,3 +130,54 @@ func TestLoginOptionFromSecret(t *testing.T) { }) } } + +func TestOIDCAdaptHelper(t *testing.T) { + auth := &authn.Basic{ + Username: "flux", + Password: "flux_password", + } + + tests := []struct { + name string + auth authn.Authenticator + expectedLogin bool + wantErr bool + }{ + { + name: "Login from basic auth with empty auth", + auth: &authn.Basic{}, + expectedLogin: false, + wantErr: false, + }, + { + name: "Login from basic auth", + auth: auth, + expectedLogin: true, + wantErr: false, + }, + { + name: "Login with missing password", + auth: &authn.Basic{Username: "flux"}, + expectedLogin: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + loginOpt, err := OIDCAdaptHelper(tt.auth) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).To(BeNil()) + + if tt.expectedLogin { + g.Expect(loginOpt).ToNot(BeNil()) + } else { + g.Expect(loginOpt).To(BeNil()) + } + }) + } +}