Enable concurrency in BMO controllers

controllers/metal3.io/baremetalhost_controller.go

@@ -1682,30 +1682,35 @@ func (r *BareMetalHostReconciler) updateEventHandler(e event.UpdateEvent) bool {
 }
 
 // SetupWithManager registers the reconciler to be run by the manager
-func (r *BareMetalHostReconciler) SetupWithManager(mgr ctrl.Manager, preprovImgEnable bool) error {
+func (r *BareMetalHostReconciler) SetupWithManager(mgr ctrl.Manager, preprovImgEnable bool, options controller.Options) error {
 
-	maxConcurrentReconciles := runtime.NumCPU()
-	if maxConcurrentReconciles > 8 {
-		maxConcurrentReconciles = 8
-	}
-	if maxConcurrentReconciles < 2 {
-		maxConcurrentReconciles = 2
-	}
-	if mcrEnv, ok := os.LookupEnv("BMO_CONCURRENCY"); ok {
-		mcr, err := strconv.Atoi(mcrEnv)
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("BMO_CONCURRENCY value: %s is invalid", mcrEnv))
+	maxConcurrentReconciles := 0
+	if options.MaxConcurrentReconciles > 0 {
+		maxConcurrentReconciles = options.MaxConcurrentReconciles
+	} else {
+		maxConcurrentReconciles = runtime.NumCPU()
+
+		if maxConcurrentReconciles > 8 {
+			maxConcurrentReconciles = 8
+		}
+		if maxConcurrentReconciles < 2 {
+			maxConcurrentReconciles = 2
 		}
-		if mcr > 0 {
-			ctrl.Log.Info(fmt.Sprintf("BMO_CONCURRENCY of %d is set via an environment variable", mcr))
-			maxConcurrentReconciles = mcr
+		if mcrEnv, ok := os.LookupEnv("BMO_CONCURRENCY"); ok {
+			mcr, err := strconv.Atoi(mcrEnv)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("BMO_CONCURRENCY value: %s is invalid", mcrEnv))
+			}
+			if mcr > 0 {
+				ctrl.Log.Info(fmt.Sprintf("BMO_CONCURRENCY of %d is set via an environment variable", mcr))
+				maxConcurrentReconciles = mcr
+			} else {
+				ctrl.Log.Info(fmt.Sprintf("Invalid BMO_CONCURRENCY value. Operator Concurrency will be set to a default value of %d", maxConcurrentReconciles))
+			}
 		} else {
-			ctrl.Log.Info(fmt.Sprintf("Invalid BMO_CONCURRENCY value. Operator Concurrency will be set to a default value of %d", maxConcurrentReconciles))
+			ctrl.Log.Info(fmt.Sprintf("Operator Concurrency will be set to a default value of %d", maxConcurrentReconciles))
 		}
-	} else {
-		ctrl.Log.Info(fmt.Sprintf("Operator Concurrency will be set to a default value of %d", maxConcurrentReconciles))
 	}
-
 	opts := controller.Options{
 		MaxConcurrentReconciles: maxConcurrentReconciles,
 	}
@@ -1716,7 +1721,7 @@ func (r *BareMetalHostReconciler) SetupWithManager(mgr ctrl.Manager, preprovImgE
 			predicate.Funcs{
 				UpdateFunc: r.updateEventHandler,
 			}).
-		WithOptions(opts).
+		WithOptions(options).
 		Owns(&corev1.Secret{})
 
 	if preprovImgEnable {

controllers/metal3.io/bmceventsubscription_controller.go

@@ -30,6 +30,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -290,9 +291,10 @@ func (r *BMCEventSubscriptionReconciler) updateEventHandler(e event.UpdateEvent)
 }
 
 // SetupWithManager registers the reconciler to be run by the manager
-func (r *BMCEventSubscriptionReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *BMCEventSubscriptionReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&metal3v1alpha1.BMCEventSubscription{}).
+		WithOptions(options).
 		WithEventFilter(predicate.Funcs{
 			UpdateFunc: r.updateEventHandler,
 		}).

controllers/metal3.io/hostfirmwaresettings_controller.go

@@ -35,6 +35,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -364,10 +365,11 @@ func (r *HostFirmwareSettingsReconciler) getOrCreateFirmwareSchema(info *rInfo,
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *HostFirmwareSettingsReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *HostFirmwareSettingsReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&metal3v1alpha1.HostFirmwareSettings{}).
+		WithOptions(options).
 		WithEventFilter(
 			predicate.Funcs{
 				UpdateFunc: r.updateEventHandler,

controllers/metal3.io/preprovisioningimage_controller.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	metal3 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
 	"github.com/metal3-io/baremetal-operator/pkg/imageprovider"
@@ -350,9 +351,10 @@ func (r *PreprovisioningImageReconciler) CanStart() bool {
 	return false
 }
 
-func (r *PreprovisioningImageReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *PreprovisioningImageReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&metal3.PreprovisioningImage{}).
+		WithOptions(options).
 		Owns(&corev1.Secret{}).
 		Complete(r)
 }

docs/configuration.md

@@ -35,15 +35,12 @@ client certificate authentication (mTLS) to be enabled.
 `IRONIC_SKIP_CLIENT_SAN_VERIFY` -- ("True", "False") Whether to skip the ironic
 client certificate SAN validation.
 
-`BMO_CONCURRENCY` -- The number of concurrent reconciles performed by the
-Operator. Default is the number of CPUs, but no less than 2 and no more than 8.
-
 `PROVISIONING_LIMIT` -- The desired maximum number of hosts that could be (de)provisioned
 simultaneously by the Operator. The limit does not apply to hosts that use
 virtual media for provisioning. The Operator will try to enforce this limit,
 but overflows could happen in case of slow provisioners and / or higher number of
 concurrent reconciles. For such reasons, it is highly recommended to keep
-BMO_CONCURRENCY value lower than the requested PROVISIONING_LIMIT. Default is 20.
+the controller concurrency value lower than the requested PROVISIONING_LIMIT. Default is 20.
 
 `IRONIC_EXTERNAL_URL_V6` -- This is the URL where Ironic will find the image for
 nodes that use IPv6. In dual stack environments, this can be used to tell Ironic which IP

main.go

@@ -27,6 +27,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
@@ -95,6 +96,10 @@ func main() {
 	var runInTestMode bool
 	var runInDemoMode bool
 	var webhookPort int
+	var baremetalhostConcurrency int
+	var preprovisioningimageConcurrency int
+	var hostfirmwaresettingsConcurrency int
+	var bmceventsubscriptionConcurrency int
 
 	// From CAPI point of view, BMO should be able to watch all namespaces
 	// in case of a deployment that is not multi-tenant. If the deployment
@@ -116,6 +121,15 @@ func main() {
 		"The address the health endpoint binds to.")
 	flag.IntVar(&webhookPort, "webhook-port", 9443,
 		"Webhook Server port (set to 0 to disable)")
+	flag.IntVar(&baremetalhostConcurrency, "baremetalhost-concurrency", 10,
+		"Number of baremetalhosts to process simultaneously")
+	flag.IntVar(&preprovisioningimageConcurrency, "preprovisioningimage-concurrency", 10,
+		"Number of preprovisioningimages to process simultaneously")
+	flag.IntVar(&hostfirmwaresettingsConcurrency, "hostfirmwaresettings-concurrency", 10,
+		"Number of hostfirmwaresettings to process simultaneously")
+	flag.IntVar(&bmceventsubscriptionConcurrency, "bmceventsubscription-concurrency", 10,
+		"Number of bmceventsubscriptions to process simultaneously")
+
 	opts := zap.Options{
 		Development: devLogging,
 		TimeEncoder: zapcore.ISO8601TimeEncoder,
@@ -170,7 +184,7 @@ func main() {
 		Log:                ctrl.Log.WithName("controllers").WithName("BareMetalHost"),
 		ProvisionerFactory: provisionerFactory,
 		APIReader:          mgr.GetAPIReader(),
-	}).SetupWithManager(mgr, preprovImgEnable); err != nil {
+	}).SetupWithManager(mgr, preprovImgEnable, concurrency(baremetalhostConcurrency)); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "BareMetalHost")
 		os.Exit(1)
 	}
@@ -184,7 +198,7 @@ func main() {
 			ImageProvider: imageprovider.NewDefaultImageProvider(),
 		}
 		if imgReconciler.CanStart() {
-			if err = (&imgReconciler).SetupWithManager(mgr); err != nil {
+			if err = (&imgReconciler).SetupWithManager(mgr, concurrency(preprovisioningimageConcurrency)); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "PreprovisioningImage")
 				os.Exit(1)
 			}
@@ -196,7 +210,7 @@ func main() {
 		Client:             mgr.GetClient(),
 		Log:                ctrl.Log.WithName("controllers").WithName("HostFirmwareSettings"),
 		ProvisionerFactory: provisionerFactory,
-	}).SetupWithManager(mgr); err != nil {
+	}).SetupWithManager(mgr, concurrency(hostfirmwaresettingsConcurrency)); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "HostFirmwareSettings")
 		os.Exit(1)
 	}
@@ -205,7 +219,7 @@ func main() {
 		Client:             mgr.GetClient(),
 		Log:                ctrl.Log.WithName("controllers").WithName("BMCEventSubscription"),
 		ProvisionerFactory: provisionerFactory,
-	}).SetupWithManager(mgr); err != nil {
+	}).SetupWithManager(mgr, concurrency(bmceventsubscriptionConcurrency)); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "BMCEventSubscription")
 		os.Exit(1)
 	}
@@ -222,3 +236,7 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func concurrency(c int) controller.Options {
+	return controller.Options{MaxConcurrentReconciles: c}
+}