Properly configure metrics-server

charts/imagevector.go

@@ -18,8 +18,8 @@
 package charts
 
 import (
-	"strings"
 	_ "embed"
+	"strings"
 
 	"github.com/gardener/gardener/pkg/utils/imagevector"
 	"k8s.io/apimachinery/pkg/util/runtime"

cmd/gardener-extension-provider-vsphere/app/app.go

@@ -162,9 +162,11 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 			reconcileOpts.Completed().Apply(&vsphereworker.DefaultAddOptions.IgnoreOperationAnnotation)
 			workerCtrlOpts.Completed().Apply(&vsphereworker.DefaultAddOptions.Controller)
 
-			if _, _, err := webhookOptions.Completed().AddToManager(mgr); err != nil {
+			_, shootWebhooks, err := webhookOptions.Completed().AddToManager(mgr)
+			if err != nil {
 				return errors.Wrap(err, "Could not add webhooks to manager")
 			}
+			vspherecontrolplane.DefaultAddOptions.ShootWebhooks = shootWebhooks
 
 			if err := controllerSwitches.Completed().AddToManager(mgr); err != nil {
 				return errors.Wrap(err, "Could not add controllers to manager")

pkg/cmd/options.go

@@ -21,13 +21,16 @@ import (
 	workercontroller "github.com/gardener/gardener-extension-provider-vsphere/pkg/controller/worker"
 	controlplanewebhook "github.com/gardener/gardener-extension-provider-vsphere/pkg/webhook/controlplane"
 	controlplaneexposurewebhook "github.com/gardener/gardener-extension-provider-vsphere/pkg/webhook/controlplaneexposure"
+	shootwebhook "github.com/gardener/gardener-extension-provider-vsphere/pkg/webhook/shoot"
+
 	controllercmd "github.com/gardener/gardener/extensions/pkg/controller/cmd"
 	extensionscontrolplanecontroller "github.com/gardener/gardener/extensions/pkg/controller/controlplane"
 	extensionshealthcheckcontroller "github.com/gardener/gardener/extensions/pkg/controller/healthcheck"
 	extensionsinfrastructurecontroller "github.com/gardener/gardener/extensions/pkg/controller/infrastructure"
 	extensionsworkercontroller "github.com/gardener/gardener/extensions/pkg/controller/worker"
 	webhookcmd "github.com/gardener/gardener/extensions/pkg/webhook/cmd"
 	extensioncontrolplanewebhook "github.com/gardener/gardener/extensions/pkg/webhook/controlplane"
+	extensionshootwebhook "github.com/gardener/gardener/extensions/pkg/webhook/shoot"
 )
 
 // ControllerSwitchOptions are the controllercmd.SwitchOptions for the provider controllers.
@@ -45,5 +48,6 @@ func WebhookSwitchOptions() *webhookcmd.SwitchOptions {
 	return webhookcmd.NewSwitchOptions(
 		webhookcmd.Switch(extensioncontrolplanewebhook.WebhookName, controlplanewebhook.AddToManager),
 		webhookcmd.Switch(extensioncontrolplanewebhook.ExposureWebhookName, controlplaneexposurewebhook.AddToManager),
+		webhookcmd.Switch(extensionshootwebhook.WebhookName, shootwebhook.AddToManager),
 	)
 }

pkg/controller/controlplane/add.go

@@ -23,6 +23,8 @@ import (
 	"github.com/gardener/gardener/extensions/pkg/controller/controlplane"
 	"github.com/gardener/gardener/extensions/pkg/controller/controlplane/genericactuator"
 	"github.com/gardener/gardener/extensions/pkg/util"
+
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -43,6 +45,8 @@ type AddOptions struct {
 	IgnoreOperationAnnotation bool
 	// GardenId is the Gardener garden identity
 	GardenId string
+	// ShootWebhooks specifies the list of desired Shoot MutatingWebhooks.
+	ShootWebhooks []admissionregistrationv1.MutatingWebhook
 }
 
 // AddToManagerWithOptions adds a controller with the given Options to the given manager.
@@ -51,7 +55,7 @@ func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) error {
 	return controlplane.Add(mgr, controlplane.AddArgs{
 		Actuator: genericactuator.NewActuator(vsphere.Name, controlPlaneSecrets, nil, configChart, controlPlaneChart, controlPlaneShootChart,
 			nil, storageClassChart, nil, NewValuesProvider(logger, opts.GardenId), extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot),
-			charts.ImageVector(), "", nil, mgr.GetWebhookServer().Port, logger),
+			charts.ImageVector(), "", opts.ShootWebhooks, mgr.GetWebhookServer().Port, logger),
 		ControllerOptions: opts.Controller,
 		Predicates:        controlplane.DefaultPredicates(opts.IgnoreOperationAnnotation),
 		Type:              vsphere.Type,

pkg/webhook/shoot/add.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shoot
+
+import (
+	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
+	"github.com/gardener/gardener/extensions/pkg/webhook/shoot"
+	appsv1 "k8s.io/api/apps/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+var (
+	// DefaultAddOptions are the default AddOptions for AddToManager.
+	DefaultAddOptions = AddOptions{}
+)
+
+// AddOptions are options to apply when adding the Equinix Metal shoot webhook to the manager.
+type AddOptions struct{}
+
+var logger = log.Log.WithName("vsphere-shoot-webhook")
+
+// AddToManagerWithOptions creates a webhook with the given options and adds it to the manager.
+func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) (*extensionswebhook.Webhook, error) {
+	logger.Info("Adding webhook to manager")
+	return shoot.New(mgr, shoot.Args{
+		Types:   []client.Object{&appsv1.Deployment{}},
+		Mutator: NewMutator(),
+	})
+}
+
+// AddToManager creates a webhook with the default options and adds it to the manager.
+func AddToManager(mgr manager.Manager) (*extensionswebhook.Webhook, error) {
+	return AddToManagerWithOptions(mgr, DefaultAddOptions)
+}

pkg/webhook/shoot/metrics_server.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shoot
+
+import (
+	"context"
+
+	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
+	appsv1 "k8s.io/api/apps/v1"
+)
+
+func (m *mutator) mutateMetricsServerDeployment(_ context.Context, deployment *appsv1.Deployment) error {
+	if c := extensionswebhook.ContainerWithName(deployment.Spec.Template.Spec.Containers, "metrics-server"); c != nil {
+		c.Command = extensionswebhook.EnsureStringWithPrefix(c.Command, "--kubelet-preferred-address-types=", "InternalIP")
+	}
+
+	return nil
+}

pkg/webhook/shoot/metrics_server_test.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shoot
+
+import (
+	"context"
+
+	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Mutator", func() {
+	Describe("#mutateMetricsServerDeployment", func() {
+		It("should correctly set the preferred address types", func() {
+			var (
+				mutator    = &mutator{}
+				deployment = &appsv1.Deployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "metrics-server",
+						Namespace: metav1.NamespaceSystem,
+					},
+					Spec: appsv1.DeploymentSpec{
+						Template: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name: "metrics-server",
+										Command: []string{
+											"/metrics-server",
+											"--authorization-always-allow-paths=/livez,/readyz",
+											"--profiling=false",
+											"--cert-dir=/home/certdir",
+											"--secure-port=8443",
+											"--kubelet-insecure-tls",
+											"--kubelet-preferred-address-types=Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP",
+											"--tls-cert-file=/srv/metrics-server/tls/tls.crt",
+											"--tls-private-key-file=/srv/metrics-server/tls/tls.key",
+										},
+									},
+								},
+							},
+						},
+					},
+				}
+			)
+
+			Expect(mutator.mutateMetricsServerDeployment(context.TODO(), deployment)).To(Succeed())
+
+			c := extensionswebhook.ContainerWithName(deployment.Spec.Template.Spec.Containers, "metrics-server")
+			Expect(c).To(Not(BeNil()))
+			Expect(c.Command).To(ContainElement("--kubelet-preferred-address-types=InternalIP"))
+		})
+	})
+})

pkg/webhook/shoot/mutator.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain m copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shoot
+
+import (
+	"context"
+
+	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
+	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+type mutator struct {
+	logger logr.Logger
+}
+
+// NewMutator creates a new Mutator that mutates resources in the shoot cluster.
+func NewMutator() extensionswebhook.Mutator {
+	return &mutator{
+		logger: log.Log.WithName("shoot-mutator"),
+	}
+}
+
+func (m *mutator) Mutate(ctx context.Context, new, old client.Object) error {
+	acc, err := meta.Accessor(new)
+	if err != nil {
+		return errors.Wrapf(err, "could not create accessor during webhook")
+	}
+	// If the object does have a deletion timestamp then we don't want to mutate anything.
+	if acc.GetDeletionTimestamp() != nil {
+		return nil
+	}
+
+	switch x := new.(type) {
+	case *appsv1.Deployment:
+		switch x.Name {
+		case "metrics-server":
+			extensionswebhook.LogMutation(logger, x.Kind, x.Namespace, x.Name)
+			return m.mutateMetricsServerDeployment(ctx, x)
+		}
+	}
+	return nil
+}

pkg/webhook/shoot/shoot_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shoot_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestShoot(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shoot Webhook Suite")
+}