Replace schemaJob with a call to management API (#44)

* Move from datastax/cass-operator to k8ssandra/cass-operator

* Add new createSchema call that uses management-api to create the schema, remove schemaJob creation

* Fix make test, add fake http server to accept schema creation calls, disable metrics-server in tests

* Remove commented out schema job check from tests

* Update to cass-operator v1.7.0

* Run e2e-test on PRs

* Separate CRD deployment in the e2e-test and wait for it to complete before deploying the operators

* Fix CRD deployment path in envtest

* Increase timeout

* Remove resource requirements to fit GHA better

* Remove dead code

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Unit testing
+name: Testing and building docker images
 
 on:
   push:
@@ -10,7 +10,7 @@ on:
 
 jobs:
   testing:
-    name: Run unit tests
+    name: Run tests
     runs-on: ubuntu-latest
     env:
       GOPATH: /home/runner/go
@@ -21,21 +21,29 @@ jobs:
         with:
           go-version: ^1.15
         id: go
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
-
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-
       - name: Run unit tests
         run: |
           export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
           make test
+      - name: Create Kind Cluster
+        uses: helm/[email protected]
+        with:
+          version: v0.10.0
+          node_image: kindest/node:v1.20.2
+          cluster_name: kind
+          config: test/config/kind/kind_config_3_workers.yaml
+      - name: Run e2e-test
+        run: |
+          export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+          make e2e-test
   push_docker_image:
     runs-on: ubuntu-latest
     needs: testing

Makefile

@@ -43,8 +43,9 @@ all: manager
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test: generate fmt vet manifests
 	mkdir -p ${ENVTEST_ASSETS_DIR}
-	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/master/hack/setup-envtest.sh
-	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh && fetch_envtest_tools $(ENVTEST_ASSETS_DIR) && setup_envtest_env $(ENVTEST_ASSETS_DIR) && go test ./controllers/... ./pkg/... -coverprofile cover.out
+	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
+	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh && fetch_envtest_tools $(ENVTEST_ASSETS_DIR) && setup_envtest_env $(ENVTEST_ASSETS_DIR) && go test -v ./controllers/... ./pkg/... 
+	# -coverprofile cover.out
 
 # Build manager binary
 manager: generate fmt vet

controllers/cassandradatacenter_controller.go

@@ -32,7 +32,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	cassdcv1beta1 "github.com/datastax/cass-operator/operator/pkg/apis/cassandra/v1beta1"
+	cassdcv1beta1 "github.com/k8ssandra/cass-operator/operator/pkg/apis/cassandra/v1beta1"
 	api "github.com/k8ssandra/reaper-operator/api/v1alpha1"
 	manager "github.com/k8ssandra/reaper-operator/pkg/reaper"
 )

controllers/cassandradatacenter_controller_test.go

@@ -3,7 +3,7 @@ package controllers
 import (
 	"context"
 	"fmt"
-	cassdcapi "github.com/datastax/cass-operator/operator/pkg/apis/cassandra/v1beta1"
+	cassdcapi "github.com/k8ssandra/cass-operator/operator/pkg/apis/cassandra/v1beta1"
 	api "github.com/k8ssandra/reaper-operator/api/v1alpha1"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

controllers/reaper_controller_test.go

@@ -11,14 +11,13 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
-	v1batch "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	cassdcapi "github.com/datastax/cass-operator/operator/pkg/apis/cassandra/v1beta1"
+	cassdcapi "github.com/k8ssandra/cass-operator/operator/pkg/apis/cassandra/v1beta1"
 )
 
 const (
@@ -77,6 +76,59 @@ var _ = Describe("Reaper controller", func() {
 			}
 			return cassdc.Status.CassandraOperatorProgress == cassdcapi.ProgressReady
 		}, timeout, interval).Should(BeTrue())
+
+		pod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-cassdc-pod1",
+				Namespace: ReaperNamespace,
+				Labels: map[string]string{
+					cassdcapi.ClusterLabel:    "test-dc1",
+					cassdcapi.DatacenterLabel: "dc1",
+				},
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  "cassandra",
+						Image: "k8ssandra/cassandra-nothere:latest",
+					},
+				},
+			},
+		}
+		Expect(k8sClient.Create(context.Background(), pod)).Should(Succeed())
+
+		podIP := "127.0.0.1"
+
+		patchPod := client.MergeFrom(pod.DeepCopy())
+		pod.Status = corev1.PodStatus{
+			PodIP: podIP,
+			PodIPs: []corev1.PodIP{
+				{
+					IP: podIP,
+				},
+			},
+		}
+		Expect(k8sClient.Status().Patch(context.Background(), pod, patchPod)).Should(Succeed())
+
+		service := &corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-dc-test-dc-all-pods-service",
+				Namespace: ReaperNamespace,
+			},
+			Spec: corev1.ServiceSpec{
+				Ports: []corev1.ServicePort{
+					{
+						Name: "mgmt-api-http",
+						Port: int32(8080),
+					},
+				},
+				Selector: map[string]string{
+					cassdcapi.ClusterLabel:    "test-dc1",
+					cassdcapi.DatacenterLabel: "dc1",
+				},
+			},
+		}
+		Expect(k8sClient.Create(context.Background(), service)).Should(Succeed())
 	})
 
 	Specify("create a new Reaper instance", func() {
@@ -88,51 +140,20 @@ var _ = Describe("Reaper controller", func() {
 		serviceKey := types.NamespacedName{Namespace: ReaperNamespace, Name: reconcile.GetServiceName(reaper.Name)}
 		service := &corev1.Service{}
 
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), serviceKey, service)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "service creation check failed")
+		Eventually(func() error {
+			return k8sClient.Get(context.Background(), serviceKey, service)
+		}, timeout, interval).Should(Succeed(), "service creation check failed")
 
 		Expect(len(service.OwnerReferences)).Should(Equal(1))
 		Expect(service.OwnerReferences[0].UID).Should(Equal(reaper.UID))
 
-		By("check that the schema job is created")
-		jobKey := types.NamespacedName{Namespace: reaper.Namespace, Name: reaper.Name + "-schema"}
-		job := &v1batch.Job{}
-
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), jobKey, job)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "schema job creation check failed")
-
-		// We need to mock the job completion in order for the deployment to get created
-		jobPatch := client.MergeFrom(job.DeepCopy())
-		job.Status.Conditions = append(job.Status.Conditions, v1batch.JobCondition{
-			Type:   v1batch.JobComplete,
-			Status: corev1.ConditionTrue,
-		})
-		Expect(k8sClient.Status().Patch(context.Background(), job, jobPatch)).Should(Succeed())
-
-		Expect(len(job.OwnerReferences)).Should(Equal(1))
-		Expect(job.OwnerReferences[0].UID).Should(Equal(reaper.GetUID()))
-
 		By("check that the deployment is created")
 		deploymentKey := types.NamespacedName{Namespace: ReaperNamespace, Name: ReaperName}
 		deployment := &appsv1.Deployment{}
 
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), deploymentKey, deployment)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "deployment creation check failed")
+		Eventually(func() error {
+			return k8sClient.Get(context.Background(), deploymentKey, deployment)
+		}, timeout, interval).Should(Succeed(), "deployment creation check failed")
 
 		Expect(len(deployment.OwnerReferences)).Should(Equal(1), "deployment owner reference not set")
 		Expect(deployment.OwnerReferences[0].UID).Should(Equal(reaper.GetUID()), "deployment owner reference has wrong uid")
@@ -187,40 +208,6 @@ var _ = Describe("Reaper controller", func() {
 		}
 		Expect(k8sClient.Create(context.Background(), service)).Should(Succeed())
 
-		By("create the schema job")
-		jobKey := types.NamespacedName{Namespace: ReaperNamespace, Name: ReaperName + "-schema"}
-		// We can use a fake job here with only the required properties set. Since the job already
-		// exists, the reconciler will just check its status. There are unit tests to verify that
-		// the job is created as expected.
-		job := &v1batch.Job{
-			ObjectMeta: metav1.ObjectMeta{
-				Namespace: jobKey.Namespace,
-				Name:      jobKey.Name,
-			},
-			Spec: v1batch.JobSpec{
-				Template: corev1.PodTemplateSpec{
-					Spec: corev1.PodSpec{
-						RestartPolicy: corev1.RestartPolicyOnFailure,
-						Containers: []corev1.Container{
-							{
-								Name:  "fake-schema-job",
-								Image: "fake-schema-job:test",
-							},
-						},
-					},
-				},
-			},
-		}
-		Expect(k8sClient.Create(context.Background(), job)).Should(Succeed())
-
-		// We need to mock the job completion in order for the deployment to get created
-		jobPatch := client.MergeFrom(job.DeepCopy())
-		job.Status.Conditions = append(job.Status.Conditions, v1batch.JobCondition{
-			Type:   v1batch.JobComplete,
-			Status: corev1.ConditionTrue,
-		})
-		Expect(k8sClient.Status().Patch(context.Background(), job, jobPatch)).Should(Succeed())
-
 		By("create the deployment")
 		// We can use a fake deployment here with only the required properties set. Since the deployment
 		// already exists, the reconciler will just check that it is ready. There are unit tests to
@@ -285,38 +272,13 @@ var _ = Describe("Reaper controller", func() {
 		}
 		Expect(k8sClient.Create(context.Background(), reaper)).Should(Succeed())
 
-		// Remove unnecessary parts, verify that the deployment has envVar set for autoscheduling
-		By("check that the schema job is created")
-		jobKey := types.NamespacedName{Namespace: reaper.Namespace, Name: reaper.Name + "-schema"}
-		job := &v1batch.Job{}
-
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), jobKey, job)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "schema job creation check failed")
-
-		// We need to mock the job completion in order for the deployment to get created
-		jobPatch := client.MergeFrom(job.DeepCopy())
-		job.Status.Conditions = append(job.Status.Conditions, v1batch.JobCondition{
-			Type:   v1batch.JobComplete,
-			Status: corev1.ConditionTrue,
-		})
-		Expect(k8sClient.Status().Patch(context.Background(), job, jobPatch)).Should(Succeed())
-
 		By("check that the deployment is created")
 		deploymentKey := types.NamespacedName{Namespace: ReaperNamespace, Name: ReaperName}
 		deployment := &appsv1.Deployment{}
 
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), deploymentKey, deployment)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "deployment creation check failed")
+		Eventually(func() error {
+			return k8sClient.Get(context.Background(), deploymentKey, deployment)
+		}, timeout, interval).Should(Succeed(), "deployment creation check failed")
 
 		Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(1))
 
@@ -349,46 +311,13 @@ var _ = Describe("Reaper controller", func() {
 		reaper.Spec.ServerConfig.CassandraBackend.CassandraUserSecretName = "top-secret-cass"
 		Expect(k8sClient.Create(context.Background(), reaper)).Should(Succeed())
 
-		By("check that the schema job is created")
-		jobKey := types.NamespacedName{Namespace: reaper.Namespace, Name: reaper.Name + "-schema"}
-		job := &v1batch.Job{}
-
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), jobKey, job)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "schema job creation check failed")
-
-		By("verify schema job has EnvVars set")
-		Expect(job.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(5))
-		Expect(job.Spec.Template.Spec.Containers[0].Env[3].Name).To(Equal("USERNAME"))
-		Expect(job.Spec.Template.Spec.Containers[0].Env[3].ValueFrom.SecretKeyRef.LocalObjectReference.Name).To(Equal("top-secret-cass"))
-		Expect(job.Spec.Template.Spec.Containers[0].Env[3].ValueFrom.SecretKeyRef.Key).To(Equal("username"))
-		Expect(job.Spec.Template.Spec.Containers[0].Env[4].Name).To(Equal("PASSWORD"))
-		Expect(job.Spec.Template.Spec.Containers[0].Env[4].ValueFrom.SecretKeyRef.LocalObjectReference.Name).To(Equal("top-secret-cass"))
-		Expect(job.Spec.Template.Spec.Containers[0].Env[4].ValueFrom.SecretKeyRef.Key).To(Equal("password"))
-
-		// We need to mock the job completion in order for the deployment to get created
-		jobPatch := client.MergeFrom(job.DeepCopy())
-		job.Status.Conditions = append(job.Status.Conditions, v1batch.JobCondition{
-			Type:   v1batch.JobComplete,
-			Status: corev1.ConditionTrue,
-		})
-		Expect(k8sClient.Status().Patch(context.Background(), job, jobPatch)).Should(Succeed())
-
 		By("check that the deployment is created")
 		deploymentKey := types.NamespacedName{Namespace: ReaperNamespace, Name: ReaperName}
 		deployment := &appsv1.Deployment{}
 
-		Eventually(func() bool {
-			err := k8sClient.Get(context.Background(), deploymentKey, deployment)
-			if err != nil {
-				return false
-			}
-			return true
-		}, timeout, interval).Should(BeTrue(), "deployment creation check failed")
+		Eventually(func() error {
+			return k8sClient.Get(context.Background(), deploymentKey, deployment)
+		}, timeout, interval).Should(Succeed(), "deployment creation check failed")
 
 		By("verify the deployment has CassAuth EnvVars")
 		envVars := deployment.Spec.Template.Spec.Containers[0].Env