diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go index 386916359a4..49701e8929c 100644 --- a/cmd/aro/operator.go +++ b/cmd/aro/operator.go @@ -23,6 +23,7 @@ import ( pkgoperator "github.com/Azure/ARO-RP/pkg/operator" aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" "github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook" + "github.com/Azure/ARO-RP/pkg/operator/controllers/autosizednodes" "github.com/Azure/ARO-RP/pkg/operator/controllers/banner" "github.com/Azure/ARO-RP/pkg/operator/controllers/checker" "github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro" @@ -210,6 +211,11 @@ func operator(ctx context.Context, log *logrus.Entry) error { if err = (muo.NewReconciler(arocli, kubernetescli, dh)).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller %s: %v", muo.ControllerName, err) } + if err = (autosizednodes.NewReconciler( + log.WithField("controller", autosizednodes.ControllerName), + mgr)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller %s: %v", autosizednodes.ControllerName, err) + } } if err = (checker.NewReconciler( diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index 1e6d88b6953..c468f4bffc0 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -76,5 +76,6 @@ func DefaultOperatorFlags() OperatorFlags { "aro.routefix.enabled": flagTrue, "aro.storageaccounts.enabled": flagTrue, "aro.workaround.enabled": flagTrue, + "aro.autosizednodes.enable": flagFalse, } } diff --git a/pkg/operator/controllers/autosizednodes/autosizednodes_controller.go b/pkg/operator/controllers/autosizednodes/autosizednodes_controller.go new file mode 100644 index 00000000000..05a0416d832 --- /dev/null +++ b/pkg/operator/controllers/autosizednodes/autosizednodes_controller.go @@ -0,0 +1,135 @@ +package autosizednodes + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + + "github.com/coreos/ignition/v2/config/util" + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/sirupsen/logrus" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" +) + +type Reconciler struct { + client client.Client + + log *logrus.Entry +} + +const ( + ControllerName = "AutoSizedNodes" + + ControllerEnabled = "aro.autosizednodes.enabled" + configName = "dynamic-node" +) + +func NewReconciler(log *logrus.Entry, mgr ctrl.Manager) *Reconciler { + return &Reconciler{ + client: mgr.GetClient(), + + log: log, + } +} + +func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + var aro arov1alpha1.Cluster + var err error + + err = r.client.Get(ctx, request.NamespacedName, &aro) + if err != nil { + err = fmt.Errorf("unable to fetch aro cluster: %w", err) + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + r.log.Infof("Config changed, autoSize: %t\n", aro.Spec.OperatorFlags.GetSimpleBoolean(ControllerEnabled)) + + // key is used to locate the object in the etcd + key := types.NamespacedName{ + Name: configName, + } + + if !aro.Spec.OperatorFlags.GetSimpleBoolean(ControllerEnabled) { + // defaults to deleting the config + config := mcv1.KubeletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + }, + } + err = r.client.Delete(ctx, &config, &client.DeleteOptions{}) + if err != nil { + err = fmt.Errorf("could not delete KubeletConfig: %w", err) + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + defaultConfig := makeConfig() + + var config mcv1.KubeletConfig + err = r.client.Get(ctx, key, &config) + if kerrors.IsNotFound(err) { + // If config doesn't exist, create a new one + err := r.client.Create(ctx, &defaultConfig, &client.CreateOptions{}) + if err != nil { + err = fmt.Errorf("could not create KubeletConfig: %w", err) + } + return ctrl.Result{}, err + } + if err != nil { + // If error, return it (controller-runtime will requeue for a retry) + return ctrl.Result{}, fmt.Errorf("could not fetch KubeletConfig: %w", err) + } + + // If already exists, update the spec + config.Spec = defaultConfig.Spec + err = r.client.Update(ctx, &config, &client.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("could not update KubeletConfig: %w", err) + } + return ctrl.Result{}, err +} + +// SetupWithManager prepares the controller with info who to watch +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + clusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool { + name := o.GetName() + return strings.EqualFold(arov1alpha1.SingletonClusterName, name) + }) + + b := ctrl.NewControllerManagedBy(mgr). + For(&arov1alpha1.Cluster{}, builder.WithPredicates(clusterPredicate)) + + // Controller adds ControllerManagedBy to KubeletConfit created by this controller. + // Any changes will trigger reconcile, but only for that config. + return b. + Named(ControllerName). + Owns(&mcv1.KubeletConfig{}). + Complete(r) +} + +func makeConfig() mcv1.KubeletConfig { + return mcv1.KubeletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + }, + Spec: mcv1.KubeletConfigSpec{ + AutoSizingReserved: util.BoolToPtr(true), + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pools.operator.machineconfiguration.openshift.io/worker": "", + }, + }, + }, + } +} diff --git a/pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go b/pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go new file mode 100644 index 00000000000..7deb7c62f50 --- /dev/null +++ b/pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go @@ -0,0 +1,129 @@ +package autosizednodes + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "reflect" + "strconv" + "testing" + + "github.com/coreos/ignition/v2/config/util" + "github.com/google/go-cmp/cmp" + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/sirupsen/logrus" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + // This "_" import is counterintuitive but is required to initialize the scheme + // ARO unfortunately relies on implicit import and its side effect for this + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + _ "github.com/Azure/ARO-RP/pkg/util/scheme" +) + +func TestAutosizednodesReconciler(t *testing.T) { + aro := func(autoSizeEnabled bool) *arov1alpha1.Cluster { + return &arov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aro", + Namespace: "openshift-azure-operator", + }, + Spec: arov1alpha1.ClusterSpec{ + OperatorFlags: arov1alpha1.OperatorFlags{ + ControllerEnabled: strconv.FormatBool(autoSizeEnabled), + }, + }, + } + } + + emptyConfig := mcv1.KubeletConfig{} + config := makeConfig() + + tests := []struct { + name string + wantGetErr error + client client.Client + wantConfig *mcv1.KubeletConfig + }{ + { + name: "is not needed", + client: fake.NewClientBuilder().WithRuntimeObjects(aro(false)).Build(), + wantConfig: &emptyConfig, + wantGetErr: kerrors.NewNotFound(mcv1.Resource("kubeletconfigs"), "dynamic-node"), + }, + { + name: "is needed and not present already", + client: fake.NewClientBuilder().WithRuntimeObjects(aro(true)).Build(), + wantConfig: &config, + wantGetErr: nil, + }, + { + name: "is needed and present already", + client: fake.NewClientBuilder().WithRuntimeObjects(aro(true), &config).Build(), + wantConfig: &config, + }, + { + name: "is not needed and is present", + client: fake.NewClientBuilder().WithRuntimeObjects(aro(false), &config).Build(), + wantConfig: &emptyConfig, + wantGetErr: kerrors.NewNotFound(mcv1.Resource("kubeletconfigs"), "dynamic-node"), + }, + { + name: "is needed and config got modified", + client: fake.NewClientBuilder().WithRuntimeObjects( + aro(true), + &mcv1.KubeletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + }, + Spec: mcv1.KubeletConfigSpec{ + AutoSizingReserved: util.BoolToPtr(false), + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pools.operator.machineconfiguration.openshift.io/worker": "", + }, + }, + }, + }).Build(), + wantConfig: &config, + wantGetErr: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + r := Reconciler{ + client: test.client, + log: logrus.NewEntry(logrus.StandardLogger()), + } + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "openshift-azure-operator", Name: "aro"}}) + if err != nil { + t.Error(err) + } + + key := types.NamespacedName{ + Name: configName, + } + var c mcv1.KubeletConfig + + err = r.client.Get(ctx, key, &c) + if err != nil && err.Error() != test.wantGetErr.Error() || err == nil && test.wantGetErr != nil { + t.Error(err) + } + + if !reflect.DeepEqual(test.wantConfig.Spec, c.Spec) { + t.Error(cmp.Diff(test.wantConfig.Spec, c.Spec)) + } + + if result != (ctrl.Result{}) { + t.Error("reconcile returned an unexpected result") + } + }) + } +} diff --git a/pkg/operator/controllers/autosizednodes/doc.go b/pkg/operator/controllers/autosizednodes/doc.go new file mode 100644 index 00000000000..89f7845670a --- /dev/null +++ b/pkg/operator/controllers/autosizednodes/doc.go @@ -0,0 +1,9 @@ +package autosizednodes + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +// autosizednodes monitors/creates/removes "dynamic-node" KubeletConfig +// that tells machine-config-operator to turn on auto sized nodes feature +// the code that is executed by the mco: +// - https://github.com/openshift/machine-config-operator/blob/fbc4d8e46a7746442f4de3651113d2181d458b12/templates/common/_base/files/kubelet-auto-sizing.yaml