feature: add autosizednodes reconciler

Introduce autosizednodes reconciler which watches aro cluster object
feature flags for ReconcileAutoSizedNodes.

When feature flag is present new KubeletConfig is created enabling the
AutoSizingReserver feature which auto computes the system reserved
for nodes.

cmd/aro/operator.go

@@ -23,6 +23,7 @@ import (
 	pkgoperator "github.com/Azure/ARO-RP/pkg/operator"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/autosizednodes"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/banner"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checker"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro"
@@ -210,6 +211,11 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 		if err = (muo.NewReconciler(arocli, kubernetescli, dh)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller %s: %v", muo.ControllerName, err)
 		}
+		if err = (autosizednodes.NewReconciler(
+			log.WithField("controller", autosizednodes.ControllerName),
+			mgr)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller %s: %v", autosizednodes.ControllerName, err)
+		}
 	}
 
 	if err = (checker.NewReconciler(

pkg/api/defaults.go

@@ -76,5 +76,6 @@ func DefaultOperatorFlags() OperatorFlags {
 		"aro.routefix.enabled":                     flagTrue,
 		"aro.storageaccounts.enabled":              flagTrue,
 		"aro.workaround.enabled":                   flagTrue,
+		"aro.autosizednodes.enable":                flagFalse,
 	}
 }

pkg/operator/controllers/autosizednodes/autosizednodes_controller.go

@@ -0,0 +1,135 @@
+package autosizednodes
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/coreos/ignition/v2/config/util"
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	"github.com/sirupsen/logrus"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+)
+
+type Reconciler struct {
+	client client.Client
+
+	log *logrus.Entry
+}
+
+const (
+	ControllerName = "AutoSizedNodes"
+
+	ControllerEnabled = "aro.autosizednodes.enabled"
+	configName        = "dynamic-node"
+)
+
+func NewReconciler(log *logrus.Entry, mgr ctrl.Manager) *Reconciler {
+	return &Reconciler{
+		client: mgr.GetClient(),
+
+		log: log,
+	}
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
+	var aro arov1alpha1.Cluster
+	var err error
+
+	err = r.client.Get(ctx, request.NamespacedName, &aro)
+	if err != nil {
+		err = fmt.Errorf("unable to fetch aro cluster: %w", err)
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	r.log.Infof("Config changed, autoSize: %t\n", aro.Spec.OperatorFlags.GetSimpleBoolean(ControllerEnabled))
+
+	// key is used to locate the object in the etcd
+	key := types.NamespacedName{
+		Name: configName,
+	}
+
+	if !aro.Spec.OperatorFlags.GetSimpleBoolean(ControllerEnabled) {
+		// defaults to deleting the config
+		config := mcv1.KubeletConfig{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: configName,
+			},
+		}
+		err = r.client.Delete(ctx, &config, &client.DeleteOptions{})
+		if err != nil {
+			err = fmt.Errorf("could not delete KubeletConfig: %w", err)
+		}
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	defaultConfig := makeConfig()
+
+	var config mcv1.KubeletConfig
+	err = r.client.Get(ctx, key, &config)
+	if kerrors.IsNotFound(err) {
+		// If config doesn't exist, create a new one
+		err := r.client.Create(ctx, &defaultConfig, &client.CreateOptions{})
+		if err != nil {
+			err = fmt.Errorf("could not create KubeletConfig: %w", err)
+		}
+		return ctrl.Result{}, err
+	}
+	if err != nil {
+		// If error, return it (controller-runtime will requeue for a retry)
+		return ctrl.Result{}, fmt.Errorf("could not fetch KubeletConfig: %w", err)
+	}
+
+	// If already exists, update the spec
+	config.Spec = defaultConfig.Spec
+	err = r.client.Update(ctx, &config, &client.UpdateOptions{})
+	if err != nil {
+		err = fmt.Errorf("could not update KubeletConfig: %w", err)
+	}
+	return ctrl.Result{}, err
+}
+
+// SetupWithManager prepares the controller with info who to watch
+func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
+	clusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
+		name := o.GetName()
+		return strings.EqualFold(arov1alpha1.SingletonClusterName, name)
+	})
+
+	b := ctrl.NewControllerManagedBy(mgr).
+		For(&arov1alpha1.Cluster{}, builder.WithPredicates(clusterPredicate))
+
+		// Controller adds ControllerManagedBy to KubeletConfit created by this controller.
+		// Any changes will trigger reconcile, but only for that config.
+	return b.
+		Named(ControllerName).
+		Owns(&mcv1.KubeletConfig{}).
+		Complete(r)
+}
+
+func makeConfig() mcv1.KubeletConfig {
+	return mcv1.KubeletConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: configName,
+		},
+		Spec: mcv1.KubeletConfigSpec{
+			AutoSizingReserved: util.BoolToPtr(true),
+			MachineConfigPoolSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"pools.operator.machineconfiguration.openshift.io/worker": "",
+				},
+			},
+		},
+	}
+}

pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go

@@ -0,0 +1,129 @@
+package autosizednodes
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/coreos/ignition/v2/config/util"
+	"github.com/google/go-cmp/cmp"
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	"github.com/sirupsen/logrus"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	// This "_" import is counterintuitive but is required to initialize the scheme
+	// ARO unfortunately relies on implicit import and its side effect for this
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
+)
+
+func TestAutosizednodesReconciler(t *testing.T) {
+	aro := func(autoSizeEnabled bool) *arov1alpha1.Cluster {
+		return &arov1alpha1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "aro",
+				Namespace: "openshift-azure-operator",
+			},
+			Spec: arov1alpha1.ClusterSpec{
+				OperatorFlags: arov1alpha1.OperatorFlags{
+					ControllerEnabled: strconv.FormatBool(autoSizeEnabled),
+				},
+			},
+		}
+	}
+
+	emptyConfig := mcv1.KubeletConfig{}
+	config := makeConfig()
+
+	tests := []struct {
+		name       string
+		wantGetErr error
+		client     client.Client
+		wantConfig *mcv1.KubeletConfig
+	}{
+		{
+			name:       "is not needed",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(false)).Build(),
+			wantConfig: &emptyConfig,
+			wantGetErr: kerrors.NewNotFound(mcv1.Resource("kubeletconfigs"), "dynamic-node"),
+		},
+		{
+			name:       "is needed and not present already",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(true)).Build(),
+			wantConfig: &config,
+			wantGetErr: nil,
+		},
+		{
+			name:       "is needed and present already",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(true), &config).Build(),
+			wantConfig: &config,
+		},
+		{
+			name:       "is not needed and is present",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(false), &config).Build(),
+			wantConfig: &emptyConfig,
+			wantGetErr: kerrors.NewNotFound(mcv1.Resource("kubeletconfigs"), "dynamic-node"),
+		},
+		{
+			name: "is needed and config got modified",
+			client: fake.NewClientBuilder().WithRuntimeObjects(
+				aro(true),
+				&mcv1.KubeletConfig{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: configName,
+					},
+					Spec: mcv1.KubeletConfigSpec{
+						AutoSizingReserved: util.BoolToPtr(false),
+						MachineConfigPoolSelector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"pools.operator.machineconfiguration.openshift.io/worker": "",
+							},
+						},
+					},
+				}).Build(),
+			wantConfig: &config,
+			wantGetErr: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			r := Reconciler{
+				client: test.client,
+				log:    logrus.NewEntry(logrus.StandardLogger()),
+			}
+			result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "openshift-azure-operator", Name: "aro"}})
+			if err != nil {
+				t.Error(err)
+			}
+
+			key := types.NamespacedName{
+				Name: configName,
+			}
+			var c mcv1.KubeletConfig
+
+			err = r.client.Get(ctx, key, &c)
+			if err != nil && err.Error() != test.wantGetErr.Error() || err == nil && test.wantGetErr != nil {
+				t.Error(err)
+			}
+
+			if !reflect.DeepEqual(test.wantConfig.Spec, c.Spec) {
+				t.Error(cmp.Diff(test.wantConfig.Spec, c.Spec))
+			}
+
+			if result != (ctrl.Result{}) {
+				t.Error("reconcile returned an unexpected result")
+			}
+		})
+	}
+}

pkg/operator/controllers/autosizednodes/doc.go

@@ -0,0 +1,9 @@
+package autosizednodes
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+// autosizednodes monitors/creates/removes "dynamic-node" KubeletConfig
+// that tells machine-config-operator to turn on auto sized nodes feature
+// the code that is executed by the mco:
+// - https://github.com/openshift/machine-config-operator/blob/fbc4d8e46a7746442f4de3651113d2181d458b12/templates/common/_base/files/kubelet-auto-sizing.yaml