From 6bc2008b7b023284c01f5e061abf62d3658c3fe0 Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Wed, 15 Dec 2021 10:26:11 +0100 Subject: [PATCH 1/6] Add cloudprovider webhook scaffold --- pkg/cmd/options.go | 3 + pkg/webhook/cloudprovider/add.go | 36 +++++++ pkg/webhook/cloudprovider/ensurer.go | 55 +++++++++++ .../webhook/cloudprovider/cloudprovider.go | 78 ++++++++++++++++ .../pkg/webhook/cloudprovider/mutator.go | 93 +++++++++++++++++++ vendor/modules.txt | 1 + 6 files changed, 266 insertions(+) create mode 100644 pkg/webhook/cloudprovider/add.go create mode 100644 pkg/webhook/cloudprovider/ensurer.go create mode 100644 vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/cloudprovider.go create mode 100644 vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/mutator.go diff --git a/pkg/cmd/options.go b/pkg/cmd/options.go index ca71ccf96..42c45bc7a 100644 --- a/pkg/cmd/options.go +++ b/pkg/cmd/options.go @@ -24,6 +24,7 @@ import ( healthcheckcontroller "github.com/gardener/gardener-extension-provider-gcp/pkg/controller/healthcheck" infrastructurecontroller "github.com/gardener/gardener-extension-provider-gcp/pkg/controller/infrastructure" workercontroller "github.com/gardener/gardener-extension-provider-gcp/pkg/controller/worker" + cloudproviderwebhook "github.com/gardener/gardener-extension-provider-gcp/pkg/webhook/cloudprovider" controlplanewebhook "github.com/gardener/gardener-extension-provider-gcp/pkg/webhook/controlplane" controlplaneexposurewebhook "github.com/gardener/gardener-extension-provider-gcp/pkg/webhook/controlplaneexposure" @@ -37,6 +38,7 @@ import ( extensionshealthcheckcontroller "github.com/gardener/gardener/extensions/pkg/controller/healthcheck" extensionsinfrastructurecontroller "github.com/gardener/gardener/extensions/pkg/controller/infrastructure" extensionsworkercontroller "github.com/gardener/gardener/extensions/pkg/controller/worker" + extensionscloudproviderwebhook "github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider" webhookcmd "github.com/gardener/gardener/extensions/pkg/webhook/cmd" extensioncontrolplanewebhook "github.com/gardener/gardener/extensions/pkg/webhook/controlplane" ) @@ -61,5 +63,6 @@ func WebhookSwitchOptions() *webhookcmd.SwitchOptions { return webhookcmd.NewSwitchOptions( webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.New), webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.New), + webhookcmd.Switch(extensionscloudproviderwebhook.WebhookName, cloudproviderwebhook.AddToManager), ) } diff --git a/pkg/webhook/cloudprovider/add.go b/pkg/webhook/cloudprovider/add.go new file mode 100644 index 000000000..99db35d63 --- /dev/null +++ b/pkg/webhook/cloudprovider/add.go @@ -0,0 +1,36 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +import ( + "github.com/gardener/gardener-extension-provider-gcp/pkg/gcp" + + extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" + "github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var logger = log.Log.WithName("gcp-cloudprovider-webhook") + +// AddToManager creates the cloudprovider webhook and adds it to the manager. +func AddToManager(mgr manager.Manager) (*extensionswebhook.Webhook, error) { + logger.Info("adding webhook to manager") + + return cloudprovider.New(mgr, cloudprovider.Args{ + Provider: gcp.Type, + Mutator: cloudprovider.NewMutator(logger, NewEnsurer(logger)), + }) +} diff --git a/pkg/webhook/cloudprovider/ensurer.go b/pkg/webhook/cloudprovider/ensurer.go new file mode 100644 index 000000000..2d694c075 --- /dev/null +++ b/pkg/webhook/cloudprovider/ensurer.go @@ -0,0 +1,55 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +import ( + "context" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider" + gcontext "github.com/gardener/gardener/extensions/pkg/webhook/context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type ensurer struct { + logger logr.Logger + client client.Client +} + +// NewEnsurer ... +func NewEnsurer(logger logr.Logger) cloudprovider.Ensurer { + return &ensurer{ + logger: logger, + } +} + +// InjectClient injects the given client into the ensurer. +func (e *ensurer) InjectClient(client client.Client) error { + e.client = client + return nil +} + +// InjectScheme injects the given scheme into the decoder of the ensurer. +func (e *ensurer) InjectScheme(_ *runtime.Scheme) error { + return nil +} + +// EnsureCloudProviderSecret ... +func (e *ensurer) EnsureCloudProviderSecret(ctx context.Context, _ gcontext.GardenContext, new, old *corev1.Secret) error { + return nil +} diff --git a/vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/cloudprovider.go b/vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/cloudprovider.go new file mode 100644 index 000000000..84223b8de --- /dev/null +++ b/vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/cloudprovider.go @@ -0,0 +1,78 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" +) + +const ( + // WebhookName is the name of the webhook. + WebhookName = "cloudprovider" +) + +var ( + logger = log.Log.WithName("cloudprovider-webhook") +) + +// Args are the requirements to create a cloudprovider webhook. +type Args struct { + Provider string + Mutator extensionswebhook.Mutator +} + +// New creates a new cloudprovider webhook. +func New(mgr manager.Manager, args Args) (*extensionswebhook.Webhook, error) { + logger := logger.WithValues("cloud-provider", args.Provider) + + types := []client.Object{&corev1.Secret{}} + handler, err := extensionswebhook.NewBuilder(mgr, logger).WithMutator(args.Mutator, types...).Build() + if err != nil { + return nil, err + } + + namespaceSelector := buildSelector(args.Provider) + logger.Info("Creating webhook") + + return &extensionswebhook.Webhook{ + Name: WebhookName, + Target: extensionswebhook.TargetSeed, + Provider: args.Provider, + Types: types, + Webhook: &admission.Webhook{Handler: handler}, + Path: WebhookName, + Selector: namespaceSelector, + }, nil +} + +func buildSelector(provider string) *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: v1beta1constants.LabelShootProvider, + Operator: metav1.LabelSelectorOpIn, + Values: []string{provider}, + }, + }, + } +} diff --git a/vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/mutator.go b/vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/mutator.go new file mode 100644 index 000000000..670849039 --- /dev/null +++ b/vendor/github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider/mutator.go @@ -0,0 +1,93 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +import ( + "context" + "fmt" + + "github.com/gardener/gardener/extensions/pkg/webhook" + gcontext "github.com/gardener/gardener/extensions/pkg/webhook/context" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +// Ensurer ensures that the cloudprovider secret conforms to the provider requirements. +type Ensurer interface { + EnsureCloudProviderSecret(ctx context.Context, gctx gcontext.GardenContext, new, old *corev1.Secret) error +} + +// NewMutator creates a new cloudprovider mutator. +func NewMutator(logger logr.Logger, ensurer Ensurer) webhook.Mutator { + return &mutator{ + logger: logger.WithName("mutator"), + ensurer: ensurer, + } +} + +type mutator struct { + client client.Client + logger logr.Logger + ensurer Ensurer +} + +// InjectClient injects the client into the ensurer. +func (m *mutator) InjectClient(client client.Client) error { + m.client = client + if _, err := inject.ClientInto(client, m.ensurer); err != nil { + return fmt.Errorf("could not inject the client into the ensurer: %w", err) + } + return nil +} + +// InjectScheme injects the manager's scheme into the ensurer. +func (m *mutator) InjectScheme(scheme *runtime.Scheme) error { + if _, err := inject.SchemeInto(scheme, m.ensurer); err != nil { + return fmt.Errorf("could not inject scheme into the ensurer: %w", err) + } + return nil +} + +// Mutate validates and if needed mutates the given object. +func (m *mutator) Mutate(ctx context.Context, new, old client.Object) error { + if new.GetDeletionTimestamp() != nil { + return nil + } + + newSecret, ok := new.(*corev1.Secret) + if !ok { + return fmt.Errorf("could not mutate: object is not of type %q", "Secret") + } + if newSecret.Name != v1beta1constants.SecretNameCloudProvider { + return nil + } + + var oldSecret *corev1.Secret + if old != nil { + oldSecret, ok = old.(*corev1.Secret) + if !ok { + return fmt.Errorf("could not mutate: old object could not be casted to type %q", "Secret") + } + } + + etcx := gcontext.NewGardenContext(m.client, new) + webhook.LogMutation(m.logger, newSecret.Kind, newSecret.Namespace, newSecret.Name) + return m.ensurer.EnsureCloudProviderSecret(ctx, etcx, newSecret, oldSecret) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3e881c631..e6a5a16fe 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,6 +99,7 @@ github.com/gardener/gardener/extensions/pkg/util github.com/gardener/gardener/extensions/pkg/util/index github.com/gardener/gardener/extensions/pkg/util/secret github.com/gardener/gardener/extensions/pkg/webhook +github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider github.com/gardener/gardener/extensions/pkg/webhook/cmd github.com/gardener/gardener/extensions/pkg/webhook/context github.com/gardener/gardener/extensions/pkg/webhook/controlplane From 44a2f7c7f0b8779e8ef55df3a6789508f41e171f Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Thu, 16 Dec 2021 16:22:04 +0100 Subject: [PATCH 2/6] Add cloudprovider secret mutation logic --- pkg/gcp/types.go | 11 ++ pkg/webhook/cloudprovider/ensurer.go | 91 +++++++++- pkg/webhook/cloudprovider/ensurer_test.go | 193 ++++++++++++++++++++++ 3 files changed, 290 insertions(+), 5 deletions(-) create mode 100644 pkg/webhook/cloudprovider/ensurer_test.go diff --git a/pkg/gcp/types.go b/pkg/gcp/types.go index 13763e567..17c871e22 100644 --- a/pkg/gcp/types.go +++ b/pkg/gcp/types.go @@ -52,6 +52,11 @@ const ( // ServiceAccountJSONMCM is the field in a machine class secret where the service account JSON is stored at. ServiceAccountJSONMCM = "serviceAccountJSON" + // ServiceAccountSecretFieldProjectID is the field in a service account secret where the project id is stored at. + ServiceAccountSecretFieldProjectID = "projectID" + // ServiceAccountSecretFieldOrganisationID is the field in a service account secret where the organisation id is stored at. + ServiceAccountSecretFieldOrganisationID = "orgID" + // CloudControllerManagerName is a constant for the name of the CloudController deployed by the worker controller. CloudControllerManagerName = "cloud-controller-manager" // CSIControllerName is a constant for the name of the CSI controller deployment in the seed. @@ -84,6 +89,12 @@ const ( MachineControllerManagerVpaName = "machine-controller-manager-vpa" // MachineControllerManagerMonitoringConfigName is the name of the ConfigMap containing monitoring stack configurations for machine-controller-manager. MachineControllerManagerMonitoringConfigName = "machine-controller-manager-monitoring-config" + + // ExtensionPurposeLabel is a label to define the purpose of a resource for the extension. + ExtensionPurposeLabel = "gcp.provider.extensions.gardener.cloud/purpose" + // ExtensionPurposeServiceAccoutSecret is the label value for a Secret resource + // that hold service account information to a corresponding GCP organisation. + ExtensionPurposeServiceAccoutSecret = "service-account-secret" ) var ( diff --git a/pkg/webhook/cloudprovider/ensurer.go b/pkg/webhook/cloudprovider/ensurer.go index 2d694c075..4dcc67526 100644 --- a/pkg/webhook/cloudprovider/ensurer.go +++ b/pkg/webhook/cloudprovider/ensurer.go @@ -16,14 +16,17 @@ package cloudprovider import ( "context" + "encoding/json" + "fmt" - "github.com/go-logr/logr" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/gardener/gardener-extension-provider-gcp/pkg/gcp" "github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider" gcontext "github.com/gardener/gardener/extensions/pkg/webhook/context" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) type ensurer struct { @@ -31,7 +34,7 @@ type ensurer struct { client client.Client } -// NewEnsurer ... +// NewEnsurer creates cloudprovider ensurer. func NewEnsurer(logger logr.Logger) cloudprovider.Ensurer { return &ensurer{ logger: logger, @@ -49,7 +52,85 @@ func (e *ensurer) InjectScheme(_ *runtime.Scheme) error { return nil } -// EnsureCloudProviderSecret ... -func (e *ensurer) EnsureCloudProviderSecret(ctx context.Context, _ gcontext.GardenContext, new, old *corev1.Secret) error { +// EnsureCloudProviderSecret ensures that cloudprovider secret contain a serviceaccount.json (if not present). +func (e *ensurer) EnsureCloudProviderSecret(ctx context.Context, _ gcontext.GardenContext, new, _ *corev1.Secret) error { + if hasSecretKey(new, gcp.ServiceAccountJSONField) { + return nil + } + + if !hasSecretKey(new, gcp.ServiceAccountSecretFieldProjectID) || !hasSecretKey(new, gcp.ServiceAccountSecretFieldOrganisationID) { + return fmt.Errorf("could not assign a service account as either project id or org id is missing") + } + + serviceAccountSecret, err := e.getManagedServiceAccountSecret(ctx, string(new.Data[gcp.ServiceAccountSecretFieldOrganisationID])) + if err != nil { + return err + } + + serviceAccountData, err := generateServiceAccountData(serviceAccountSecret.Data[gcp.ServiceAccountJSONField], string(new.Data[gcp.ServiceAccountSecretFieldProjectID])) + if err != nil { + return err + } + new.Data[gcp.ServiceAccountJSONField] = serviceAccountData + return nil } + +func hasSecretKey(secret *corev1.Secret, key string) bool { + if _, ok := secret.Data[key]; ok { + return true + } + return false +} + +func (e *ensurer) getManagedServiceAccountSecret(ctx context.Context, orgID string) (*corev1.Secret, error) { + var ( + serviceAccountSecretList = corev1.SecretList{} + matchingSecrets = []*corev1.Secret{} + labelSelector = client.MatchingLabels{gcp.ExtensionPurposeLabel: gcp.ExtensionPurposeServiceAccoutSecret} + ) + + if err := e.client.List(ctx, &serviceAccountSecretList, labelSelector); err != nil { + return nil, err + } + + for _, sec := range serviceAccountSecretList.Items { + if !hasSecretKey(&sec, gcp.ServiceAccountSecretFieldOrganisationID) { + continue + } + + if string(sec.Data[gcp.ServiceAccountSecretFieldOrganisationID]) == orgID { + tmp := &sec + matchingSecrets = append(matchingSecrets, tmp) + } + } + + if len(matchingSecrets) == 0 { + return nil, fmt.Errorf("found no service account secret matching to org id %q", orgID) + } + + if len(matchingSecrets) > 1 { + return nil, fmt.Errorf("found more than one service account secret matching to org id %q", orgID) + } + + if !hasSecretKey(matchingSecrets[0], gcp.ServiceAccountJSONField) { + return nil, fmt.Errorf("service account secret does not contain service account information") + } + + return matchingSecrets[0], nil +} + +func generateServiceAccountData(serviceAccountTemplate []byte, projectID string) ([]byte, error) { + var servieAccountData map[string]string + if err := json.Unmarshal(serviceAccountTemplate, &servieAccountData); err != nil { + return nil, err + } + + servieAccountData["project_id"] = projectID + + servieAccountDataRaw, err := json.Marshal(servieAccountData) + if err != nil { + return nil, err + } + return servieAccountDataRaw, nil +} diff --git a/pkg/webhook/cloudprovider/ensurer_test.go b/pkg/webhook/cloudprovider/ensurer_test.go new file mode 100644 index 000000000..4305d7697 --- /dev/null +++ b/pkg/webhook/cloudprovider/ensurer_test.go @@ -0,0 +1,193 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider_test + +import ( + "context" + "testing" + + . "github.com/gardener/gardener-extension-provider-gcp/pkg/webhook/cloudprovider" + + "github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider" + gcontext "github.com/gardener/gardener/extensions/pkg/webhook/context" + mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CloudProvider Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + logger = log.Log.WithName("gcp-cloudprovider-webhook-test") + ctx = context.TODO() + ensurer cloudprovider.Ensurer + + ctrl *gomock.Controller + c *mockclient.MockClient + + secret *corev1.Secret + serviceAccountData string + serviceAccountSecret corev1.Secret + + gctx = gcontext.NewGardenContext(nil, nil) + labelSelector = client.MatchingLabels{"gcp.provider.extensions.gardener.cloud/purpose": "service-account-secret"} + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + c = mockclient.NewMockClient(ctrl) + ensurer = NewEnsurer(logger) + + err := ensurer.(inject.Client).InjectClient(c) + Expect(err).NotTo(HaveOccurred()) + + secret = &corev1.Secret{ + Data: map[string][]byte{ + "projectID": []byte("gcp-project-id"), + "orgID": []byte("gcp-project-id"), + }, + } + + serviceAccountData = `{"key":"test"}` + serviceAccountSecret = corev1.Secret{ + Data: map[string][]byte{ + "projectID": []byte("gcp-project-id"), + "orgID": []byte("gcp-project-id"), + "serviceaccount.json": []byte(serviceAccountData), + }, + } + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureCloudProviderSecret", func() { + It("should pass as a service account is present", func() { + secret.Data["serviceaccount.json"] = []byte("some-service-account-data") + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should fail as secret does not contain a project id", func() { + delete(secret.Data, "projectID") + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).To(HaveOccurred()) + }) + + It("should fail as secret does not contain a organisation id", func() { + delete(secret.Data, "orgID") + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).To(HaveOccurred()) + }) + + It("should add service account", func() { + c.EXPECT().List(gomock.Any(), &corev1.SecretList{}, labelSelector). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + list.Items = []corev1.Secret{serviceAccountSecret} + return nil + }) + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Data).To(Equal(map[string][]byte{ + "projectID": []byte("gcp-project-id"), + "orgID": []byte("gcp-project-id"), + "serviceaccount.json": []byte(`{"key":"test","project_id":"gcp-project-id"}`), + })) + }) + + It("should add service account, but not consider one of the service account secrets as contain no organisation id", func() { + serviceAccountSecret2 := serviceAccountSecret.DeepCopy() + delete(serviceAccountSecret2.Data, "orgID") + + c.EXPECT().List(gomock.Any(), &corev1.SecretList{}, labelSelector). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + list.Items = []corev1.Secret{serviceAccountSecret, *serviceAccountSecret2} + return nil + }) + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Data).To(Equal(map[string][]byte{ + "projectID": []byte("gcp-project-id"), + "orgID": []byte("gcp-project-id"), + "serviceaccount.json": []byte(`{"key":"test","project_id":"gcp-project-id"}`), + })) + }) + + It("should fail as no matching service account secrets exists", func() { + c.EXPECT().List(gomock.Any(), &corev1.SecretList{}, labelSelector). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + list.Items = []corev1.Secret{} + return nil + }) + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).To(HaveOccurred()) + }) + + It("should fail as more than one matching service account secret exists", func() { + serviceAccountSecret2 := serviceAccountSecret.DeepCopy() + + c.EXPECT().List(gomock.Any(), &corev1.SecretList{}, labelSelector). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + list.Items = []corev1.Secret{serviceAccountSecret, *serviceAccountSecret2} + return nil + }) + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).To(HaveOccurred()) + }) + + It("should fail as the matching service account secret contains no service account information", func() { + delete(serviceAccountSecret.Data, "serviceaccount.json") + + c.EXPECT().List(gomock.Any(), &corev1.SecretList{}, labelSelector). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + list.Items = []corev1.Secret{serviceAccountSecret} + return nil + }) + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).To(HaveOccurred()) + }) + + It("should fail as the service account cannot be calculated due to invalid input", func() { + serviceAccountSecret.Data["serviceaccount.json"] = []byte("invalid-service-account-json-input") + + c.EXPECT().List(gomock.Any(), &corev1.SecretList{}, labelSelector). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + list.Items = []corev1.Secret{serviceAccountSecret} + return nil + }) + + err := ensurer.EnsureCloudProviderSecret(ctx, gctx, secret, nil) + Expect(err).To(HaveOccurred()) + }) + }) +}) From 826f0cd71c62c7f1c2f12434638360d7d89f39d8 Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Fri, 17 Dec 2021 10:08:11 +0100 Subject: [PATCH 3/6] Allow gcp credentials secret without serviceaccount.json --- pkg/apis/gcp/validation/secret.go | 35 +++++++++++++++++++----- pkg/apis/gcp/validation/secret_test.go | 38 +++++++++++++++----------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/pkg/apis/gcp/validation/secret.go b/pkg/apis/gcp/validation/secret.go index d9ba734aa..ed1c8ea5e 100644 --- a/pkg/apis/gcp/validation/secret.go +++ b/pkg/apis/gcp/validation/secret.go @@ -24,21 +24,42 @@ import ( var projectIDRegexp = regexp.MustCompile(`^(?P[a-z][a-z0-9-]{4,28}[a-z0-9])$`) -// ValidateCloudProviderSecret checks whether the given secret contains a valid GCP service account. +// ValidateCloudProviderSecret checks whether the given secret contains a valid GCP service account +// or a valid project id and an organisation id. func ValidateCloudProviderSecret(secret *corev1.Secret) error { - serviceAccountJSON, ok := secret.Data[gcp.ServiceAccountJSONField] - if !ok { - return fmt.Errorf("missing %q field in secret", gcp.ServiceAccountJSONField) + if serviceAccountJSON, ok := secret.Data[gcp.ServiceAccountJSONField]; ok { + return validateServiceAccountJSON(serviceAccountJSON) } - projectID, err := gcp.ExtractServiceAccountProjectID(serviceAccountJSON) + if !hasSecretKey(secret, gcp.ServiceAccountSecretFieldProjectID) || !hasSecretKey(secret, gcp.ServiceAccountSecretFieldOrganisationID) { + return fmt.Errorf("missing required field(s). Either field %q or the fields %q and %q must be present", gcp.ServiceAccountJSONField, gcp.ServiceAccountSecretFieldProjectID, gcp.ServiceAccountSecretFieldOrganisationID) + } + + return validateProjectID(string(secret.Data[gcp.ServiceAccountSecretFieldProjectID])) +} + +func validateServiceAccountJSON(jsonData []byte) error { + projectID, err := gcp.ExtractServiceAccountProjectID(jsonData) if err != nil { return err } - if !projectIDRegexp.MatchString(projectID) { - return fmt.Errorf("service account project ID does not match the expected format '%s'", projectIDRegexp) + if err := validateProjectID(projectID); err != nil { + return fmt.Errorf("invalid service account field: %w", err) } + return nil +} +func validateProjectID(projectID string) error { + if !projectIDRegexp.MatchString(projectID) { + return fmt.Errorf("project ID does not match the expected format '%s'", projectIDRegexp) + } return nil } + +func hasSecretKey(secret *corev1.Secret, key string) bool { + if _, ok := secret.Data[key]; ok { + return true + } + return false +} diff --git a/pkg/apis/gcp/validation/secret_test.go b/pkg/apis/gcp/validation/secret_test.go index 55fd586d5..8ce1ebdd4 100644 --- a/pkg/apis/gcp/validation/secret_test.go +++ b/pkg/apis/gcp/validation/secret_test.go @@ -19,7 +19,6 @@ import ( "strings" . "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp/validation" - "github.com/gardener/gardener-extension-provider-gcp/pkg/gcp" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" @@ -39,24 +38,31 @@ var _ = Describe("Secret validation", func() { Expect(err).To(matcher) }, - Entry("should return error when the serviceaccount.json field is missing", - map[string][]byte{}, HaveOccurred()), - Entry("should return error when the project ID is missing", - map[string][]byte{gcp.ServiceAccountJSONField: []byte(`{"foo": "bar"}`)}, HaveOccurred()), - Entry("should return error when the project ID starts with a digit", - map[string][]byte{gcp.ServiceAccountJSONField: []byte(`{"project_id": "0my-project"}`)}, + Entry("should succeed when valid serviceaccount.json is present", + map[string][]byte{"serviceaccount.json": []byte(`{"project_id": "my-project"}`)}, + BeNil()), + Entry("should return error when serviceaccount.json does not contain a project ID", + map[string][]byte{"serviceaccount.json": []byte(`{"foo": "bar"}`)}, HaveOccurred()), + Entry("should return error when the project ID of the serviceaccount.json starts with a digit", + map[string][]byte{"serviceaccount.json": []byte(`{"project_id": "0my-project"}`)}, HaveOccurred()), - Entry("should return error when the project ID ends with hyphen", - map[string][]byte{gcp.ServiceAccountJSONField: []byte(`{"project_id": "my-project-"}`)}, + Entry("should return error when the project ID of the serviceaccount.json ends with hyphen", + map[string][]byte{"serviceaccount.json": []byte(`{"project_id": "my-project-"}`)}, HaveOccurred()), - Entry("should return error when the project ID is too short", - map[string][]byte{gcp.ServiceAccountJSONField: []byte(`{"project_id": "foo"}`)}, + Entry("should return error when the project ID of the serviceaccount.json is too short", + map[string][]byte{"serviceaccount.json": []byte(`{"project_id": "foo"}`)}, HaveOccurred()), - Entry("should return error when the project ID is too long", - map[string][]byte{gcp.ServiceAccountJSONField: []byte(fmt.Sprintf(`{"project_id": "%s"}`, strings.Repeat("a", 31)))}, + Entry("should return error when the project ID of the serviceaccount.json is too long", + map[string][]byte{"serviceaccount.json": []byte(fmt.Sprintf(`{"project_id": "%s"}`, strings.Repeat("a", 31)))}, HaveOccurred()), - Entry("should succeed when the project ID is valid", - map[string][]byte{gcp.ServiceAccountJSONField: []byte(`{"project_id": "my-project"}`)}, - BeNil()), + + Entry("should succeed when projectID and orgID are present", + map[string][]byte{"projectID": []byte("gcp-project-id"), "orgID": []byte("gcp-org-id")}, BeNil()), + Entry("should return error when no serviceaccount.json or projectID plus orgID are present", + map[string][]byte{}, HaveOccurred()), + Entry("should return error when no serviceaccount.json and only a projectID is present", + map[string][]byte{"projectID": []byte("gcp-project-id")}, HaveOccurred()), + Entry("should return error as no serviceaccount.json and only a orgID is present", + map[string][]byte{"orgID": []byte("gcp-org-id")}, HaveOccurred()), ) }) From da8ee1c4e1d79d17cecd0dc3da11859c4a806806 Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Fri, 17 Dec 2021 10:11:42 +0100 Subject: [PATCH 4/6] Avoid dry of hasSecretKey function --- pkg/apis/gcp/validation/secret.go | 10 ++----- pkg/utils/utils.go | 27 +++++++++++++++++++ pkg/utils/utils_suite_test.go | 27 +++++++++++++++++++ pkg/utils/utils_test.go | 39 ++++++++++++++++++++++++++++ pkg/webhook/cloudprovider/ensurer.go | 16 ++++-------- 5 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 pkg/utils/utils.go create mode 100644 pkg/utils/utils_suite_test.go create mode 100644 pkg/utils/utils_test.go diff --git a/pkg/apis/gcp/validation/secret.go b/pkg/apis/gcp/validation/secret.go index ed1c8ea5e..cee242ac2 100644 --- a/pkg/apis/gcp/validation/secret.go +++ b/pkg/apis/gcp/validation/secret.go @@ -19,6 +19,7 @@ import ( "regexp" "github.com/gardener/gardener-extension-provider-gcp/pkg/gcp" + "github.com/gardener/gardener-extension-provider-gcp/pkg/utils" corev1 "k8s.io/api/core/v1" ) @@ -31,7 +32,7 @@ func ValidateCloudProviderSecret(secret *corev1.Secret) error { return validateServiceAccountJSON(serviceAccountJSON) } - if !hasSecretKey(secret, gcp.ServiceAccountSecretFieldProjectID) || !hasSecretKey(secret, gcp.ServiceAccountSecretFieldOrganisationID) { + if !utils.HasSecretKey(secret, gcp.ServiceAccountSecretFieldProjectID) || !utils.HasSecretKey(secret, gcp.ServiceAccountSecretFieldOrganisationID) { return fmt.Errorf("missing required field(s). Either field %q or the fields %q and %q must be present", gcp.ServiceAccountJSONField, gcp.ServiceAccountSecretFieldProjectID, gcp.ServiceAccountSecretFieldOrganisationID) } @@ -56,10 +57,3 @@ func validateProjectID(projectID string) error { } return nil } - -func hasSecretKey(secret *corev1.Secret, key string) bool { - if _, ok := secret.Data[key]; ok { - return true - } - return false -} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 000000000..267b76728 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,27 @@ +/* + * Copyright 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * + */ + +package utils + +import corev1 "k8s.io/api/core/v1" + +// HasSecretKey checks if a given secret contain a given key. +func HasSecretKey(secret *corev1.Secret, key string) bool { + if _, ok := secret.Data[key]; ok { + return true + } + return false +} diff --git a/pkg/utils/utils_suite_test.go b/pkg/utils/utils_suite_test.go new file mode 100644 index 000000000..c1ae865b2 --- /dev/null +++ b/pkg/utils/utils_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestWorker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Utils Suite") +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go new file mode 100644 index 000000000..e974f6b47 --- /dev/null +++ b/pkg/utils/utils_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + . "github.com/gardener/gardener-extension-provider-gcp/pkg/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + gomegatypes "github.com/onsi/gomega/types" + + corev1 "k8s.io/api/core/v1" +) + +var _ = Describe("Utils", func() { + + DescribeTable("#HasSecretKey", + func(secret *corev1.Secret, key string, matcher gomegatypes.GomegaMatcher) { + Expect(HasSecretKey(secret, key)).To(matcher) + }, + Entry("should return true as secret does contain key", + &corev1.Secret{Data: map[string][]byte{"key-a": []byte("test")}}, "key-a", BeTrue()), + Entry("should return false as secret does not contain key", + &corev1.Secret{Data: map[string][]byte{"key-a": []byte("test")}}, "key-b", BeFalse()), + ) +}) diff --git a/pkg/webhook/cloudprovider/ensurer.go b/pkg/webhook/cloudprovider/ensurer.go index 4dcc67526..c3b073df8 100644 --- a/pkg/webhook/cloudprovider/ensurer.go +++ b/pkg/webhook/cloudprovider/ensurer.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/gardener/gardener-extension-provider-gcp/pkg/gcp" + "github.com/gardener/gardener-extension-provider-gcp/pkg/utils" "github.com/gardener/gardener/extensions/pkg/webhook/cloudprovider" gcontext "github.com/gardener/gardener/extensions/pkg/webhook/context" @@ -54,11 +55,11 @@ func (e *ensurer) InjectScheme(_ *runtime.Scheme) error { // EnsureCloudProviderSecret ensures that cloudprovider secret contain a serviceaccount.json (if not present). func (e *ensurer) EnsureCloudProviderSecret(ctx context.Context, _ gcontext.GardenContext, new, _ *corev1.Secret) error { - if hasSecretKey(new, gcp.ServiceAccountJSONField) { + if utils.HasSecretKey(new, gcp.ServiceAccountJSONField) { return nil } - if !hasSecretKey(new, gcp.ServiceAccountSecretFieldProjectID) || !hasSecretKey(new, gcp.ServiceAccountSecretFieldOrganisationID) { + if !utils.HasSecretKey(new, gcp.ServiceAccountSecretFieldProjectID) || !utils.HasSecretKey(new, gcp.ServiceAccountSecretFieldOrganisationID) { return fmt.Errorf("could not assign a service account as either project id or org id is missing") } @@ -76,13 +77,6 @@ func (e *ensurer) EnsureCloudProviderSecret(ctx context.Context, _ gcontext.Gard return nil } -func hasSecretKey(secret *corev1.Secret, key string) bool { - if _, ok := secret.Data[key]; ok { - return true - } - return false -} - func (e *ensurer) getManagedServiceAccountSecret(ctx context.Context, orgID string) (*corev1.Secret, error) { var ( serviceAccountSecretList = corev1.SecretList{} @@ -95,7 +89,7 @@ func (e *ensurer) getManagedServiceAccountSecret(ctx context.Context, orgID stri } for _, sec := range serviceAccountSecretList.Items { - if !hasSecretKey(&sec, gcp.ServiceAccountSecretFieldOrganisationID) { + if !utils.HasSecretKey(&sec, gcp.ServiceAccountSecretFieldOrganisationID) { continue } @@ -113,7 +107,7 @@ func (e *ensurer) getManagedServiceAccountSecret(ctx context.Context, orgID stri return nil, fmt.Errorf("found more than one service account secret matching to org id %q", orgID) } - if !hasSecretKey(matchingSecrets[0], gcp.ServiceAccountJSONField) { + if !utils.HasSecretKey(matchingSecrets[0], gcp.ServiceAccountJSONField) { return nil, fmt.Errorf("service account secret does not contain service account information") } From 4d1e5cfed088c21958a88dee8cd396a0bf959b7b Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Fri, 17 Dec 2021 16:49:14 +0100 Subject: [PATCH 5/6] Add service account secret to extension chart --- .../templates/secret-serviceaccount.yaml | 14 ++++++++++++++ charts/gardener-extension-provider-gcp/values.yaml | 6 ++++++ example/controller-registration.yaml | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 charts/gardener-extension-provider-gcp/templates/secret-serviceaccount.yaml diff --git a/charts/gardener-extension-provider-gcp/templates/secret-serviceaccount.yaml b/charts/gardener-extension-provider-gcp/templates/secret-serviceaccount.yaml new file mode 100644 index 000000000..f4b0ba5b0 --- /dev/null +++ b/charts/gardener-extension-provider-gcp/templates/secret-serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- range .Values.serviceAccounts }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: service-account-{{ print .orgID .serviceAccount | sha256sum | trunc 12 }} + namespace: {{ $.Release.Namespace }} + labels: + gcp.provider.extensions.gardener.cloud/purpose: service-account-secret +data: + orgID: {{ .orgID }} + serviceaccount.json: {{ .serviceAccount }} +type: Opaque +{{- end }} diff --git a/charts/gardener-extension-provider-gcp/values.yaml b/charts/gardener-extension-provider-gcp/values.yaml index c706720dc..3d6975e0d 100644 --- a/charts/gardener-extension-provider-gcp/values.yaml +++ b/charts/gardener-extension-provider-gcp/values.yaml @@ -78,3 +78,9 @@ config: gardener: seed: provider: gcp + +# serviceAccounts: +# - orgID: b3JnLTE= +# serviceAccount: ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgImNsaWVudF9pZCI6ICIxMjM0Igp9 +# - orgID: b3JnLTI= +# serviceAccount: ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgImNsaWVudF9pZCI6ICI1Njc4Igp9 diff --git a/example/controller-registration.yaml b/example/controller-registration.yaml index e97be8b2a..bcb5d91b5 100644 --- a/example/controller-registration.yaml +++ b/example/controller-registration.yaml @@ -5,7 +5,7 @@ metadata: name: provider-gcp type: helm providerConfig: - chart: H4sIAAAAAAAAA+09YXPbNrL9rF+B0b3OJb2IEiXZvtNN3nuKraaeJrbGcpPpXO95KBKSWFMEC5BydEn/+9sFQAqkKFFyXLtphUlGEri7WAC7i8ViQU8d7tGQ8gb9ENNQ+CxsRJwtfA+qpm7U/OoBSgvKydGR/IRS/JTf7U7Xbh+1j4+x3j46Oep8RY4eovGqkojY4YR8xRmLt8FVPf9Cy7Ri/k9nDo+tpTMP7t8GTvBxt7tx/tvwLD//JyfwQVoP183N5U8+/07kv6Mc571HFnbNiSLjp9VuWa2GRxc1jwqX+1EsH/TJdzSYExdFg0wYJ/GMktdakMjr0yEZahEimVTVQmdOe6RC3GqLssafeoz+yKVK/z3mWlP2eW1U6H/bPmkX9b91ZB/0/zFKs0lOWbTk/nQWk2fuc9Ju2f8go/6QjAYEVNsJ5Q9nMvED34kpcdk8csKlRfpBQCSaIJwKyhfUs8j1zBcEQCmBz8B3QaSoR5IQbQFaiX7kuPAxYpP4zuGUvFEgL8jCIm2wFi6NYuIIErIY8Big8DtfALVQor85Px1cAGPYQq3ZhH8phZJGMtraopG21SLPEKCuH9Wf/xNJLFlC5s4SGyUJNBZnndAMQevYbRiA0KXkzo9nihtFxUIaP2oabBw7AO4AQgS/JiYgcWLNtCyzOI56zebd3Z3lSI4txqdNPWiiqfvaAK411g9hQAWO9i+Jz6HH4yUBew0Izhh4DZw7OWFTTuFZzJDrO+7Hfjh9QYQecCTj+SLm/jiJc4OW8ghdNwFg2EAE6v0ROR/Vyav+6Hz0Aom8P7/+7vKHa/K+f3XVv7g+H4zI5RU5vbw4O78+v7yAX9+S/sWP5Pvzi7MXhPo4kzCcEcceAJs+DidIDNIaUZpjIV1SRERdf+K70LVwmjhTSqYMFogQekQiyue+wGkVwKCHZAJ/7sdOLKvW+mXVAGTKelM0dijHltWEfwsaeow3p8BfMrZgkpupQVx9mTnubTPFa7gsjDkLArCPnE5xpGSTlpgR03ASS7fwbnA1ggHRv+gHBzpOm5uooKNFXkGDSfQqcW9p3ENaqmIAKEv9W8iVGL+fKkpDGCMqK84uRlfUZdzrTRmbBsBxwBLPg0E5DyccMHnixglXwO8Zv6Ucv+L4kCE0hOOslnYaomAJYg6bSKKI6WVfV+J04EhDm5y6MVn1jeT6VotM6odV/XdRqtb/mIK8wsSLz9gJ7r//OznB9f+w//vty+7zfzOjAVhdYcXRnnvBCv/P7nQK+792u33SPfh/j1E+fmwQj078ELwi3KLVSePXX2tV2zTEgpVTwtZMEoEzpoEAfyaybulSEZM/kjEs3BTkyPJZExvK0dhAYuEEiebo40fwZ9wg8TI+LaIRtzCyjltkEKn0yAYI3b5sab0XfgiiAw6hRLeuaEAd8DMugLlSzjLW/DksgoozQvCJPyEzRww5PP9A6mLmtI+Oe9DsO2wemkJ4K3amJMOIuB/GE1L/Wvzv16IIyWnEhB8zvtxGAvpIywj27k0QOmv0uzghHo0CtpzTMNZ7/Ew4RHNhb55I4IyBF7t0A0eIHKoAl9lLAnA/rNu/yxkp0Hlq3foSyu72H9y6iT+dO1FDTv4CXD3GG+iQ4y6DbokRVsX/usedgv0/sTvtg/1/jKLtT06v38mpvUxnVlm/XJjw1g+9Hm49QCLeOlFtTmPHc2KnB7ZABfrK7XW56GgkAfuDEmMqq5WZUaa5V2LQkfwnqARZjkkXoVN2ZIviJi+nPfIJiWztdZ6cYdaeesoetNxH/wM2naLV3fVUoEL/O8etov/XBQfwoP+PUR5KsbVMNJSU5DVayZihvxj9yuRNWKkMWjJSoQUtUVGDHtGUM4We+EEMwmn4YoiAGo1k//Xt+ZvrwdW/a0QX6Y8VSuRwQXkG8taJ3VkBxCD/jWTLDPzMnRAMBv8GFKSx8WFG/nu6vClyAZ3Kng8lN+vNw9TIsCof5tm9UtHWmzMYkLTumie09psNwNxxZ+CMbRqCLY//HIOw+VHOmP6RRsMVfsPjPrBi9FophPB/1x2dM8+fLHfuaAYIRtHz0SQZgNg9uUm8mSMZKghuif5vhf8T7CH5TxZ0+Sc0U3g2ANb2J+vZJrPRsP72Pzf4f5tt+VQ6+nlU4T/PWO97XnEUyM1NanZv5kkQg8cTOmF843s3N4RFGNdl/J8Jjv0XNrRblDEdoGfbrdanHfX5oQb4Kdf/e/h/e2eDVJ//Fvd/7eNW5+D/PUZ5KP8vk47fdDOnWsm2cKhmjUZDfpodAcG1Uim2DEdToxf8zYXtBNHMsSWZbAC04p+azmitsFvW9NzAB04BMoQtJFox2T/gtlDf0/bCcfFUGduAx9fLiAo5UNm5br2CvrVOAI9tU/x6FX9l+JplOcRp7Z5cGZj7sWMiZnz8Eu07KoCxX7uIkLU3TriI92xR4uzXpkLJBxSweRq7XiofAlYGWGx62fIiY58XqfoVmkBMS6NYGeSqY4CO6QV+vKzG1oApf+uiLsHH8jhaAZGY/Yhn1lvgMlU+zvf7qU3foXy1z/q/CuDv6wBsX//tVufELqz/nZZ9iP88SjGXzfQsRq2BZ9l87+wF7Lj2y9MwM+g8DRmnsLVjCXfTtcgJQ6azeZQh5Onz4vqt0HukHsNGsJ43rPs4GZhuhE1xuvBxQL7z0S4u32BaUY+05BOZbSVyhlRXnrIkjFWjAjqNsWTFt9y+vNmNj2NNAFNm6HSpOw6OiB9Of4hgCrJFYe58+CF0Fo4fYI5OjqH8o9VCAPWjhE/XgGWlAku1XXNuTLv0W4pTgmXHA4RsLZpR91YkcyOcjJux0nOBnJg9k0eU5L+sa82j9QoEa+jEM1Lf6Wyq/lyOtDpeBQ5MrgqL8QZGt7q+92C2gq0dZffvKUYqv1jSU9PTledQdaavfQ/KF75L+66L8nyxXdlTf1FtvdO2G1U2QhU5U3kofTK+DjZMgmDIQM3yPow6hY6yh7n5Y3PYs3srQW2Q5m5DgKCNBtgtrJaqjICp+WkEzL19aTChAAcazkrh3gCYyQ/SVA7RWCb2NUDzUHTchHOYRqCPP/yAipd5L01vRIRlYlsrzNEydEV5SxQzBu/bkESubkdmIt6nDZH6/tvoK11pYDLuyyY4lc1yodI61TT2h0Uy2HKESZL782piVzIsYLmYqu3iPVoysKta8kLBZZ7n/s1kqFVtzKgTxDNpCvdvxUCuasfPZabu31Qev7I16TA0VBAO52m1sG1sQKJcphj9DKFI+04m1O7fA4VXxfkdHc8Yu03VInO2Xm6Js2zE1qa+gfm8JlsaTEU+LA01xKTfanoYIa0kR3kZNc8X6K8YQdfcYOnHq8iMgCXwZ+aHpP6ivomWbruM0Hv9aAMVGi7MpUOtaG8G/bPB1c3gzeAUk9xvLvpvB6Nh/3SQQRIiY9bfcjbvGZV4akkD74pO8rW6Hh2DXuZuWdm03tfJSvk9f9t/PXgHzF5e3Vy+G1y9vzq/XuO1R5oyIdvIkWiWJk1s85Zw0sX6gKUSombdaDnzGlAScmv6LuKC/g2LmcuCHrk+HRZjFTy/k1Alq+yVRC1WGJ9IqP0qu1USppGjxoJkTt+if1TS5ewAPC1zBFQzXL2Cfe6Mb8qvKWNmbdYNOA5OzWUYgLuFm6rNM4/3gkIMLRmS4PXD2O+vPSBZlOwsAe94OsrS986lcdXVgw/UTcxoqeqedHxHuY2V0SvcYg3U5ZL87iRFv6XLjVmeWR5oAYtkxzSgSeHaQ6k8a01hYztkk5oIMYtYwKbL75HHej7HdMZELIVEYyjZW3O1C8LjpvF7k7udw/dp8ejESYL4LfMAr9tu6Ud7SeZucrk/v1VyvoX3P2IW1/3L7vE/UGxYOnkiLwGPE29Kdw0EVuV/HnVPivlfJ93D/c9HKVqbpzF5hhGLsqDWc2IXU0Ajud1uLuwx+CxpwHDIvLNMQF5JAXnoyOH9AnolsTKRjCs7/NmBvC/B2uyh/2Yi/l5HAFX631nT/87R0eH9D49STK02Zbr82oVUVa3tZoSxRM8r3/Sg9h52dgWwNg3Y2AnO1MrdIxMnEDT/4ol62qg6EV1/90TWVi93GdeqP/Uw/27L7vrPx457vxfBVOV/H9nHBf23jw7vf3icgukzpg2Qs+wk8Qz07D8qDrm6XKWTYwIYM8qvWED3Wd/3Wbl5EuAmp4FZPa85SyK542lsThqv5Tb4CGpGy8V6TROmPU7MBxjv9mlJTR5Uhq3zP0wAVw2O/mEEjktqTLwsJFv8aQLlw5yldSa4iinmvq8eg00f66FCR03usX2hvtyhpyO/Rdm3RB4/rk/JpmPZ9RlRKZReVptnov5NfZ24y2AI/NCUw3W60k8sUHM5LeV2P4JpvfRB1TPTOBbPiMpH9a5yCOt1+YHH71quUoyN2qgQPP0KjNxrFkyAyDc0ZlN3M8dbFH42JzBSgf+fVIZgwx1qXRIURjjOZFrFDjRU6EXMTwFXWSMpogwn53446rAxp0CgJHStYgw2wg+nqn4FsfboZzZWX2DPuvrSVInnIIhJLN9aoUONufsmuk1oks3TcZJXUf30aZXM6uRhSziRVITSMUfMalIw80y4jnHFdZ0S0Ihh4xJAD1NwNV9F4p9l6F+p4f3N7D00oYPj6RBs4bCWpWkaK1EFP7Dh+xm0VC4qCnmUO+l+mP3pU6/m+5fd/T+trPdwAav2fyd2cf/X7rQO938fpZTmf2vV+JPlfcUy5VnblPNhLv6zW95WdgrWkF8rzrUKp6r5Iy15MOBwcCL2OiHbe/731n+9WO9jBqr0v91dy/887h7yPx+lbNP/dGl80iDuUw/QH7zsof/qksD+4d/q8x+7eP+r02kfH/T/MYo+/6G/ZMtLtsoKSr3sGhWpgzTUi+dA6cWR4s5hpOo3xYb3u8Syj7WQ7CJvlFc2Y8DKfgUBu3snz9cHHyJHh5Fl+kPkcOAm1jmuyk+IvIYQ3hdxxrOt7K7/i8i553vAK/Tf7hbf/9e2u/ZB/x+lFLI5cJLVaz+9gqrX14MR+vy3Xdda/06HIobM62ehiN2vkELbO3oQqcdewny6WzATwvN1aldhJKBBpW+m1GSPVPb7X7/5a5ZYMvfDPpoJauSVA60o2XBxcJ0ja0XCArzyy4PVaLkUojmdM768FwsK9T5caMzi69/SXUuW6lh2wQjr1y4ZVeX8AIAKoJrTqGpUmo9h7pF1E9hawX0pB/OPVKrsv8py+7w/AFF1/mevvf+zZXcO+79HKeoWjjSS6Usee4Qm1tTlaOEz8QChwPjp6q3Yq4OFZuEOTexMe0T6DKjUkXF153xyweIhvi0e9L1mptz0iF0rpunIGn1j7aj1NSKkkW/y8dca6DjyrVerLFG1zNivm21pso9a81regh533/qVdqaOi1u9VjMS1RHIPF7M1hgzmR8azMDk5ZqtUGKVA1sKYZ4kbgMzbpRsAcvOG7fAGPc5tkDlzyO3AKoTyW2UNt25SHNDaut3AnrkX/+uFTL8ZV0h1piR+AspS2DFl0v8haQvD+zJ72kqa+QkQt1SkE6CfEaIontlKJHxTvlVTq/5dRywcXPuYLSxOU78wGtK0s0zBiLE5Z84ULRN1Uz1Ur7a/WZ1+U7hNpy5d9zVaFIP6x2rVdcV2R9XsS3btj582b2y13pV/++X2LO2emBZVq2WC5X2atn1RhVT7XY7uiq9h2C32ketWi1/pa9nWBW81ddLD4hrbka3/FUfZS/60H+xAYGaPwsWZt5o9tKNUgj5Ogy7pRKZ9bsq7A7+XL05ovDeCHf97qeKnDcmjohToOzdEO2j176uzO2i8wnpU7ww5NWyN0SrQU0Na7oSyFew/Ol9q0M5lEM5lEM5lEM5lEM5lN9b+X+/vld/AHgAAA== + chart: H4sIAAAAAAAAA+09f3PbNrL9W58Co7vOJX0RJeqHfdVN3jvFVlNdHdtjuc7LXe95KBKiGFMES5B2dEm/+9sFQAqkKFFyEjtphUnGEri7WAC7i8ViQblW5NCARg36LqYB91jQCCN26zlQ5dph85tPUFpQDns98RdK8a/4bHa6ZrvXPjjAerN32Ot8Q3qfovGqkvDYigj5JmIs3gRX9fwrLW7F/B/NrCg2Ftbcv38bOMEH3e7a+W/Ds/z8Hx7CH9L6dN1cX/7g82+F3hWNcN775NasWWGofTXaLaPVcOhtzaHcjrwwFg8G5Efqz4mNokGmLCLxjJKXSpDIy6Nzcq5EiGRSVQusOe2TCnGr3ZY1/thj9HsuVfrvMNtw2ce1UaH/bfOwXdT/Vs/c6/9DlGaTHLFwEXnuLCZP7Kek3TK/J+PBORkPCai2FYgv1nTq+Z4VU2KzeWgFC4MMfJ8INE4iyml0Sx2DXM48TgCUEvjrezaIFHVIEqAtQCsxCC0b/ozZNL6zIkpOJMgzcmuQNlgLm4YxsTgJWAx4DFCiO48DtUCgn4yOhqfAGLZQazbhX0qhpJGMtrJopG20yBMEqKtH9ad/QxILlpC5tcBGSQKNxVknFEPQOnYbBiCwKbnz4pnkRlIxkMYbRYNNYgvALUAI4dtUByRWrJgWZRbHYb/ZvLu7MyzBscEit6kGjTdVXxvAtcL6OfApx9H+NfEi6PFkQcBeA4I1AV59605MmBtReBYz5Pou8mIvcJ8RrgYcyTgejyNvksS5QUt5hK7rADBsIAL1wZiMxnXyYjAejZ8hkdejyx/Pfr4krwcXF4PTy9FwTM4uyNHZ6fHocnR2Ct9+IIPTN+Sn0enxM0I9nEkYzjDCHgCbHg4nSAzSGlOaYyFdUnhIbW/q2dC1wE0slxKXwQIRQI9ISKO5x3FaOTDoIBnfm3uxFYuqlX4ZNQBxWd9FY4dybBhN+HdLA4dFTRf4SyYGTHIzNYjLDzPLvmmmeA2bBXHEfB/sY0RdHCnRpMFnRDecxFAtXA0vxjAg6ht9Z0HHaXMdFXS0yAtoMAlfJPYNjftIS1YMAWWhvnOxEuPnI0npHMaIiorj0/EFtVnk9F3GXB849lniODAoo2AaAWaU2HESSeDXLLqhEX7E8SHn0BCOs1zaaYCCxYk+bDwJQ6aWfVWJ04EjDW1G1I7Jsm8k17daqFPfr+pfRKla/2MK8goTzz9iJ7j7/u/wENf//f7v85ft5/96Rn2wutyIwx33ghX+n9npFPZ/7Xb7sLv3/x6ivH/fIA6degF4RbhFq5PGb7/VqrZpiAUrp4Ct6SR8a0J9Dv5MaNzQhSQmviQTWLgpyJHhsSY2lKOxhsSt5SeKo/fvwZ+x/cTJ+DSIQtzAyCpukUGk0idrIFT7oqXVXngBiA44hALduKA+tcDPOAXmSjnLWPPmsAhKzgjBJ96UzCx+HsHzd6TOZ1a7d9CHZq+weWgK4Y3YckmGEUZeEE9J/Vv+9295ETKiIeNezKLFJhLQR1pGsH9vgtBZrd/FCXFo6LPFnAax2uNnwsGbt+b6iQTOGHixC9u3OM+hcnCZncQH98O4+auYkQKdx9atr6Fsb//BrZt67twKG2Lyb8HVY1EDHXLcZdANMcKq+F/3oFOw/4dmp723/w9RlP3J6fWVmNqzdGal9cuFCW+8wOnj1gMk4pUV1uY0thwrtvpgC2Sgr9xel4uOQuKwPygxpqJamhlpmvslBh3Jf4BKkOWYdBE6ZUe0yK/zctonH5DIxl7nyWlm7bGn7JOW++i/z1wXre62pwIV+t85aBX9vy44gHv9f4jyqRRbyURDSkleo6WMafqL0a9M3riRyqAhIhVK0BIZNegTRTlT6KnnxyCcmi+GCKjRSPZfP4xOLocX/64RVYQ/ViihFXEaZSCvrNieFUA08t8JtvTAz9wKwGBE34GCNNY+zMj/RBfXRS6gU9nzc8HNavMwNSKsGp3n2b2Q0dbrYxiQtO4ySmjtsw3A3LJn4IytG4INj/8Yg7D+Uc6Y/p5Gw+Zew4k8YEXrtVQI7n3RHZ0zx5sutu5oBghG0fHQJGmA2D2xSbyeIxnKCW6J/m+J/wvsIaNfDOjyL2im8GwArO0vxpN1ZqNh/Nf/XOP/TbblQ+no51G59zRjfeA4xVEg19ep2b2eJ34MHk9gBfG151xfExZiXJdFf0tw7L+yod2gjOkAPdlstT5sqc+faoAfc/2/h/+3czZI9flvcf/XPmh19v7fQ5RP5f9l0vFZN3OylWwLh2rWaDTEX70jILhGKsWG5mgq9IK/eWtafjizTEEmGwCl+Ee6M1or7JYVPdv3gFOADGALiVZM9A+4LdT3lb2wbDxVxjbg8eUipFwMVHauW6+gb6wSwGPbFL9exV8ZvmJZDHFauyNXGuZu7OiIGR+/hruOCmDs1i4iZO1NkojHO7YocHZrU6LkAwrYPI1tJ5UPDisDLDb9bHkRsc/TVP0KTSCmoVCMDHLZMUDH9AIvXlRjK8CUv1VRF+ATcRwtgUjM3uCZ9Qa4TJUP8v1+bNO3L9/ssv4vA/i7OgCb13+z1Tk0C+t/p2Xu4z8PUvRlMz2LkWvgcTbfW3sBW6794jRMDzq7AYsobO1YEtnpWmQFAVPZPNIQRunz4vot0fukHsNGsJ43rLs4GZhuhE1F9NbDAfnRQ7u4OMG0oj5piSci24rnDKmqPGJJEMtGOXQaY8mSb7F9OdmOjwNFAFNmqLtQHQdHxAvcn0OYgmxRmFvvfg6sW8vzMUcnx1D+0XIhgPpxErkrwKJSgqXarjjXpl34LcUpwbLlAUK2Fs2ofcOTuRZOxs1Y6blATsyeiCNK8mfjUvFovADBOrfiGalvdTZVfypGWh6vAgc6V4XFeA2jG13fezBbwdaWsvvXFCOVXyzpqenR0nOoOtNXvgeNbj2bDmwb5fl0s7Kn/qLceqdtN6pshCxipvJQ6mR8Few88f1zBmqW92HkKXSYPczNH5vDnt1ZCmqDNLcbAgRtNMBuYbVQZQRMzU/DZ/bNc40JCThUcEYKdwJgOj9IUzpEE5HY1wDNQ9GxkyiCaQT6+MXzKX+e99LURoQbOraxxBwvApuXt0QxY/C+DQnk6nZEJuJ92uCp77+JvtSVBibjPm+CU9ksFyqlU01tf1gkgy2HmCS5O686diXDHJYLV24X79GShl3VkhPwSOR57t5MhlrVxoxafjwTpnD3VjTkqna8XGbq7k3l8StbEw5DQwbhcJ6WC9vaBgTKWYoxyBCKtO9EQu3uPZB4VZzf0cmMsZtULTJn6/mGOMtabGXqG5jPq7OlwGTkw1BQ55j0W00PI6SV5GhURs3xOPorWtA1N1jq8TIyw2EJfMu8gNSf1dfRUm2XEXqtHq2hQoNbfemQK9rJcHA8vLgengyPMMn9+nTwajg+HxwNM0hCRMz6h4jN+1olnlpS37mg03ytqkfHoJ+5W0Y2rfd1slJ+R68GL4dXwOzZxfXZ1fDi9cXocoXXPmmKhGwtR6JZmjSxyVvCSeerA5ZKiJx1reXMa0BJyK3p24gL+jcsZjbz++Ty6LwYq4jyOwlZssp+SdRiifGBBMqvMlslYRoxasxP5vQV+kclXc4OwNMyR0A5w9Ur2MfO+Lr8mjJmVmZdg4vAqTkLfHC3cFO1fubxXlCAoSVNEpxBEHuDlQcki5IdJ+Adu+MsfW8kjKuqHr6jdqJHS2X3hOM7zm2stF7hFmsoL5fkdycp+g1drM3yzPJAC1gkO6YBTQpWHgrlWWkKG9sim1RHiFnIfOYufkIe6/kc0xnjsRAShSFlb8XVLgiPncbvde62Dt+nxaFTK/HjV8wBvG67pR7tJJnbyeXu/FbJ+Qbef49ZXPcv28f/QLFh6YwScQl4kjgu3TYQWJX/2eseFvO/Drv7+58PUpQ2uzF5ghGLsqDWU2IWU0BDsd1u3poT8FnSgOE5c44zAXkhBORTRw7vF9AriZXxZFLZ4Y8O5H0N1mYH/dcT8Xc6AqjS/86K/nd6vf37Hx6k6Fqty3T5tQuhqkrb9QhjiZ5XvulB7j3M7ApgzfXZxPKP5crdJ1PL5zT/4ol62qg8EV1990TWVj93GdeoP/Ywf7Fle/2PJpZ9vxfBVOV/98yDgv6bvf37Hx6mYPqMbgPELFtJPAM9+4+MQy4vV6nkGB/GjEYXzKe7rO+7rNxR4uMmp4FZPS8jloRix9NYnzRey23wEVSPlvPVmiZMe5zoDzDe7dGSmjyoCFvnv+gAthwc9UULHJfU6HhZSLb4VQfKhzlL63RwGVPMfV4+Bps+UUOFjprYY3tcfrhDT0d8CrNPiTh+XJ2SdceyqzMiUyidrDbPRP27+ipxm8EQeIEuh6t0hZ9YoGZHtJTb3Qim9cIHlc9041g8Iyof1bvKIazXxR88fldylWKs1UaJ4KhXYORes6ADhJ6mMeu6mznevPC1OYWR8r3/pDIEG+5A6RKnMMJxJtMydqCgAidkXgq4zBpJEUU4OffFkoeNOQUCJaErFROwEV7gyvolxMqjt2wiP8CedfmhKRPPQRCTWLy1QoUac/dNVJvQJJun4ySuonrp0yqZVcnDBrdCoQilY46Y1aRg5hm3Le2K6yoloBHDxsWHHqbgcr6KxD/K0L+Qw/vZ7D00oYLj6RBs4LCWpWlqK1EFP7DhewtaKhYViTzOnXR/mv3pY6/mu5ft/T+p8o28zm7lEFbGfw6L/l/P7Pb2/t9DFIxRRFbg0iwSks8AEWcoRduRWYixkIkSq5CeLSoxwdQZmRKD77gaHZNCK7kEmA945hDYxGyXKd6fqyJDGKpZlwBeyBwLkyhkvIRdKetZ2rngWaq9ZF9Fh3Ka8JarPXSxbwAci2zqs9D6NaFfVlhoF/0XvbrHFrBK/w/NYvyn3Wnt7/8/SFmj1mKq/2B5n1JJlU8xOs/Ff7fL28xOwRviY8W5diGrIn+kLQ4GrQg2ETudkO88/zvr/y4LvypV+t/uruR/H3T3+d8PUjbpf+oaP+ohzmMP0O+87KD/8pLQ7sc/1f6/Wbz/2em0D/b6/xBFnf/SX7PlJVtlOaVO5kWTOkhDvXgOnF4cK0YOxrJ+3dnQbpfYdrEWgl3kjUaVzWiwol++z+6uRH7N8F1oqWMkkf4UWhFwE6scd+knhE6Dc+fLcubvUbbX/9vQuufvAFTov9ktvv+zbXbNvf4/SClkc+Eky9f+OgVVr68GI1X+R7uutP5KhSLPmTPIQpHbXyGHtrf0IFKPvYT5dLegXwjJ18ldhZaACpWenlKXPZK3X/7y3V+yxLK5FwzQTFDtXgnQCpM1F4dXOTKWJAzAK788XI2WSyGc0zmLFvdiQaLehwuFWXz9Y7pryVKdyy4YYv3KJcOqnD8AkAco+jTKGpnmp5l7ZF0HNpZwX0tizgOVKvsvs1w/7gdgKs//24Xff2jjmrC3/w9R5C08YSTTl7z2CU0M147QwmfiAUKB5yfLt+Ivo6rNwh262HL7RPgMqNShdnVvND1l8Tn+WgToe01PuesTs1ZM0xM16sZqr/UtIqQnX+T9bzXQceRbrVZZonqZsV8128Jk91rzWt6CHnRfeZV2po6LW71W0y6qIJCeXpCtMfplHmgwAxOX6zZC8WUOfCmEnkmwCUy7UbYBLMs32ACj3efaAJXPR9gAKDMSNlFad+cqzQ2rrd4J6pN//btWuOEj6gqxxozEn0hZAju+XOZPJH15aF98TlPZQyvh8paScBLEM0Ik3QtNibTflFjm9OsfJz6bNOcWRhubk8TznaYg3TxmIEKR+IkTSVtXzVQvxU87XC8v30rchjV3DroKTehhvWO06qoi+3El0zBN493X3StzpVf1/36OPWvLB4Zh1Gq5UGl/eWYjY6rdbkdVpfeQzFa716rV8ld6+5pVwVu9/TRBpGZndMtf9VP2oh/1iy0I1MQjo8wbzV66UwohXodjtuRFBvWuGrODX5dvjim8N8ZevfstI+eNqcXjFCh7N0y799JTlblddP5CiosXBp1a9oZ4OaipYU1XAvEKJlSuwmEialIjPU6bdP4RnFwOn4vpygPCEnTH3NHRP1r0f1/4o7fwOTj17eCf4Zv21fTN69O3k85V4hyN+JE3cEfzU269hu8/fB/+82h0MDoavXv19lVr5IbfrzY4+iwNmqdv7a5o8LFX9H3Zl33Zl33Zl+ry/38vb5EAeAAA values: image: tag: v1.20.0-dev From 4f64496e4b1ffffacc7f8e9e53032fe0f986a615 Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Mon, 20 Dec 2021 12:36:49 +0100 Subject: [PATCH 6/6] Add docs for managed service account usage --- docs/usage-as-end-user.md | 21 +++++++++++++++++- docs/usage-as-operator.md | 45 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/docs/usage-as-end-user.md b/docs/usage-as-end-user.md index 3780f19be..273a1fa36 100644 --- a/docs/usage-as-end-user.md +++ b/docs/usage-as-end-user.md @@ -11,7 +11,7 @@ Every shoot cluster references a `SecretBinding` which itself references a `Secr The `SecretBinding` is configurable in the [Shoot cluster](https://github.com/gardener/gardener/blob/master/example/90-shoot.yaml) with the field `secretBindingName`. The required credentials for the GCP project are a [Service Account Key](https://cloud.google.com/iam/docs/service-accounts#service_account_keys) to authenticate as a [GCP Service Account](https://cloud.google.com/compute/docs/access/service-accounts). -A service account is a special account that can be used by services and applications to interact with Google Cloud Platform APIs. +A service account is a special account that can be used by services and applications to interact with Google Cloud Platform APIs. Applications can use service account credentials to authorize themselves to a set of APIs and perform actions within the permissions granted to the service account. Make sure to [enable the Google Identity and Access Management (IAM) API](https://cloud.google.com/service-usage/docs/enable-disable). @@ -41,6 +41,25 @@ data: ⚠️ Depending on your API usage it can be problematic to reuse the same Service Account Key for different Shoot clusters due to rate limits. Please consider spreading your Shoots over multiple Service Accounts on different GCP projects if you are hitting those limits, see https://cloud.google.com/compute/docs/api-rate-limits. +### Managed Service Accounts + +The operators of the Gardener GCP extension can provide managed service accounts. +This eliminates the need for users to provide an own service account for a Shoot. + +To make use of a managed service account, the GCP secret of a Shoot cluster must contain an `orgID` and a `projectID` field, but no `serviceaccount.json` field. +- The `projectID` field contains the id of the GCP project +- The `orgID` contains the id of the GCP organisation where the GCP project belongs to + +Removing the `serviceaccount.json` field and adding the `projectID` and `orgID` from the secret of an existing Shoot will also let it adopt the managed service account. + +Based on the `orgID` field, the Gardener extension will try to assign the managed service account to the Shoot. +If no managed service account can be assigned then the next operation on the Shoot will fail. + +⚠️ The managed service account need to be assigned to the users GCP project with [proper permissions](#gcp-provider-credentials) before using it. + +In addition to use this feature the user project need to have the organisation policy enabled that allow the assignment of service accounts originated in a different project of the same organisation. +More information are available [here](https://cloud.google.com/iam/docs/impersonating-service-accounts#binding-to-resources). + ## `InfrastructureConfig` The infrastructure configuration mainly describes how the network layout looks like in order to create the shoot worker nodes in a later step, thus, prepares everything relevant to create VMs, load balancers, volumes, etc. diff --git a/docs/usage-as-operator.md b/docs/usage-as-operator.md index 170ef055d..e4854cd58 100644 --- a/docs/usage-as-operator.md +++ b/docs/usage-as-operator.md @@ -89,7 +89,7 @@ The location/region where the backups will be stored defaults to the region of t The region of the backup can be different from where the seed cluster is running. However, usually it makes sense to pick the same region for the backup bucket as used for the Seed cluster. -Please find below an example `Seed` manifest (partly) that configures backups using Google Cloud Storage buckets. +Please find below an example `Seed` manifest (partly) that configures backups using Google Cloud Storage buckets. ```yaml --- @@ -113,6 +113,47 @@ An example of the referenced secret containing the credentials for the GCP Cloud #### Permissions for GCP Cloud Storage -Please make sure the service account associated with the provided credentials has the following IAM roles. +Please make sure the service account associated with the provided credentials has the following IAM roles. - [Storage Admin](https://cloud.google.com/storage/docs/access-control/iam-roles) +## Miscellaneous + +### Gardener managed Service Accounts + +The operators of the Gardener GCP extension can provide a list of managed service accounts (technical users) that can be used for GCP Shoots. +This eliminates the need for users to provide own service account for their clusters. + +GCP service accounts are always bound to one project. +But there is an option to assign a service account originated in a different project of the same GCP organisation to a project. +Based on this approach Gardener operators can provide a project which contains managed service accounts and users could assign service accounts from this project with proper permissions to their projects. + +To use this feature the user project need to have the organisation policy enabled that allow the assignment of service accounts originated in a different project of the same organisation. +More information are available [here](https://cloud.google.com/iam/docs/impersonating-service-accounts#binding-to-resources). + +In case the user provide an own service account in the Shoot secret, this one will be used instead of the managed one provided by the operator. + +Each managed service account will be maintained in a `Secret` like that: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: service-account-my-org + namespace: extension-provider-gcp + labels: + gcp.provider.extensions.gardener.cloud/purpose: service-account-secret +data: + orgID: base64(my-gcp-org-id) + serviceaccount.json: base64(my-service-account-json-without-project-id-field) +type: Opaque +``` + +The user needs to provide in its Shoot secret a `orgID` and `projectID`. + +The managed service account will be assigned based on the `orgID`. +In case there is a managed service account secret with a matching `orgID`, this one will be used for the Shoot. +If there is no matching managed service account secret then the next Shoot operation will fail. + +One of the benefits of having managed service account is that the operator controls the lifecycle of the service account and can rotate its secrets. + +After the service account secret has been rotated and the corresponding secret is updated, all Shoot clusters using it need to be reconciled or the last operation to be retried.