[#301] Replace leader-for-life leader election with leader-for-lease

.github/workflows/go.yml

@@ -24,10 +24,10 @@ jobs:
     - name: Build
       run: make build
     - name: Set up Minikube
-      uses: manusa/actions-setup-minikube@v2.9.0
+      uses: manusa/actions-setup-minikube@v2.10.0
       with:
         minikube version: v1.31.2
-        kubernetes version: v1.24.0
+        kubernetes version: v1.25.0
         driver: docker
     - name: Load WildFly images used for testing
       run: |-

config/manager/kustomization.yaml

@@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/wildfly/wildfly-operator
+  newName: localhost:5000/wildfly-operator
   newTag: latest

config/rbac/kustomization.yaml

@@ -2,8 +2,8 @@ resources:
 - service_account.yaml
 - role.yaml
 - role_binding.yaml
-#- leader_election_role.yaml
-#- leader_election_role_binding.yaml
+- leader_election_role.yaml
+- leader_election_role_binding.yaml
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.

config/rbac/leader_election_role.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: leader-election-role
+  name: wildfly-op-leader-role
 rules:
 - apiGroups:
   - ""

config/rbac/leader_election_role_binding.yaml

@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: leader-election-rolebinding
+  name: wildfly-op-leader-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: leader-election-role
+  name: wildfly-op-leader-role
 subjects:
 - kind: ServiceAccount
-  name: default
-  namespace: system
+  name: wildfly-operator
+

main.go

@@ -17,19 +17,19 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"github.com/RHsyseng/operator-utils/pkg/utils/openshift"
 	"go.uber.org/zap/zapcore"
 	"os"
 	goruntime "runtime"
+	"time"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	route "github.com/openshift/api/route/v1"
-	"github.com/operator-framework/operator-lib/leader"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	wildflyv1alpha1 "github.com/wildfly/wildfly-operator/api/v1alpha1"
 	"github.com/wildfly/wildfly-operator/controllers"
@@ -90,13 +90,8 @@ func main() {
 	}
 	setupLog.Info("Watching namespace", "namespace", namespace)
 
-	if enableLeaderElection {
-		err = leader.Become(context.TODO(), "t3dv81741s.wildfly-operator-lock")
-		if err != nil {
-			setupLog.Error(err, "Failed to retry for leader lock")
-			os.Exit(1)
-		}
-	}
+	leaseDuration := 30 * time.Second
+	renewDeadline := 20 * time.Second
 
 	setupLog.Info("Starting the Manager")
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -105,7 +100,10 @@ func main() {
 		Port:                   9443,
 		Namespace:              namespace,
 		HealthProbeBindAddress: probeAddr,
-		LeaderElection:         false,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "8kr4rta7hq.wildfly-operator-lock",
+		LeaseDuration:          &leaseDuration,
+		RenewDeadline:          &renewDeadline,
 		Logger:                 ctrl.Log.WithName("manager"),
 	})
 	if err != nil {

test/e2e/suite_test.go

@@ -27,7 +27,6 @@ import (
 	wildflyv1alpha1 "github.com/wildfly/wildfly-operator/api/v1alpha1"
 	"github.com/wildfly/wildfly-operator/controllers"
 	"io"
-	"io/ioutil"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -37,14 +36,15 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/yaml"
 	"strings"
 	"testing"
 	"time"
@@ -66,8 +66,8 @@ var (
 	testEnv        *envtest.Environment
 	operator       *appsv1.Deployment
 	serviceAccount *corev1.ServiceAccount
-	role           *rbacv1.Role
-	roleBinding    *rbacv1.RoleBinding
+	roles          []*rbacv1.Role
+	roleBindings   []*rbacv1.RoleBinding
 )
 
 const (
@@ -94,7 +94,7 @@ var _ = BeforeSuite(func() {
 
 		log.Printf("Start testing deploying the Operator")
 		testEnv = &envtest.Environment{
-			UseExistingCluster:       pointer.BoolPtr(true),
+			UseExistingCluster:       ptr.To(true),
 			CRDDirectoryPaths:        []string{filepath.Join("..", "..", "config", "crd", "bases")},
 			CRDInstallOptions:        envtest.CRDInstallOptions{CleanUpAfterUse: true},
 			ErrorIfCRDPathMissing:    true,
@@ -104,7 +104,7 @@ var _ = BeforeSuite(func() {
 		cfg, k8sClient = initialSetup()
 
 		// load resources for tests generated by kustomize
-		data, err := ioutil.ReadFile("../../dry-run/test-resources.yaml")
+		data, err := os.ReadFile("../../dry-run/test-resources.yaml")
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -118,20 +118,29 @@ var _ = BeforeSuite(func() {
 			case "ServiceAccount":
 				serviceAccount = obj.(*corev1.ServiceAccount)
 			case "Role":
-				role = obj.(*rbacv1.Role)
+				roles = append(roles, obj.(*rbacv1.Role))
 			case "RoleBinding":
-				roleBinding = obj.(*rbacv1.RoleBinding)
+				roleBindings = append(roleBindings, obj.(*rbacv1.RoleBinding))
 			default:
 				// unexpected type, ignore
 			}
 		}
 
+		// Print the deployment YAML to the console to helps debugging
+		deploymentYAML, err := yaml.Marshal(operator)
+		if err != nil {
+			fmt.Println("Error converting deployment to YAML:", err)
+			return
+		}
+
+		// Print the deployment YAML
+		fmt.Println(string(deploymentYAML))
 	} else {
 
 		// When we are running the test in local, we do not deploy the Operator
 		// web run the manager directly from the test suite
 		testEnv = &envtest.Environment{
-			UseExistingCluster: pointer.BoolPtr(true),
+			UseExistingCluster: ptr.To(true),
 			CRDDirectoryPaths: []string{
 				filepath.Join("..", "..", "config", "crd", "bases"),
 			},
@@ -146,7 +155,7 @@ var _ = BeforeSuite(func() {
 		os.Setenv("OPERATOR_NAME", "wildfly-operator")
 		os.Setenv("JBOSS_BOOTABLE_HOME", "/opt/jboss/container/wildfly-bootable-jar-server")
 
-		// start the manager and reconciler
+		// start the manager and reconcile
 		k8sManager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 			Scheme:                 scheme.Scheme,
 			MetricsBindAddress:     "0.0.0.0:8383",

test/e2e/wildflyserver_test.go

@@ -46,14 +46,19 @@ var _ = Describe("WildFly Server tests", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			log.Printf("Creating the Role")
-			roleHolder := role.DeepCopy()
-			err = k8sClient.Create(ctx, roleHolder)
-			Expect(err).ToNot(HaveOccurred())
+			for _, role := range roles {
+				log.Printf("Creating the Role %s", role.Name)
+				roleHolder := role.DeepCopy()
+				err = k8sClient.Create(ctx, roleHolder)
+				Expect(err).ToNot(HaveOccurred())
+			}
 
-			log.Printf("Creating the Role Binding")
-			rbHolder := roleBinding.DeepCopy()
-			err = k8sClient.Create(ctx, rbHolder)
-			Expect(err).ToNot(HaveOccurred())
+			for _, roleBindings := range roleBindings {
+				log.Printf("Creating the Role Binding %s", roleBindings.Name)
+				rbHolder := roleBindings.DeepCopy()
+				err = k8sClient.Create(ctx, rbHolder)
+				Expect(err).ToNot(HaveOccurred())
+			}
 
 			log.Printf("Creating the operator")
 			opHolder := operator.DeepCopy()
@@ -97,6 +102,7 @@ var _ = Describe("WildFly Server tests", func() {
 
 		log.Printf("Creating %s resource", server.Name)
 		Expect(k8sClient.Create(ctx, server)).Should(Succeed())
+
 		WaitUntilReady(ctx, k8sClient, server)
 
 		log.Printf("Scalling the server to 2 replicas")