Annotation to suspend reconcile on a CR (#404)

This adds the ability to suspend all reconcile activity against a custom
resource. This aids in debugging because the operator uses an infinity
retry model (i.e. keep retrying the reconcile and use an exponential
back-off for repeated failures). This works great unless you want to dig
into a specific behaviour that you are seeing.

If any of the CRs (VerticaDB, VerticaAutoscaler, EventTrigger) have the
`vertica.com/pause` annotation set to `true`, then no further reconcile
will get scheduled for the CR.

This also swaps the create DB parameter --force-cleanup-on-failure with
--force-removal-at-creation. It has the same affect, but if you have the
operator paused you can see the left over log files from a failed
create.

changes/unreleased/Added-20230523-142408.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: Ability to pause the reconciler for individual CRs
+time: 2023-05-23T14:24:08.039581226-03:00
+custom:
+  Issue: "404"

pkg/controllers/et/eventtrigger_controller.go

@@ -38,6 +38,7 @@ import (
 	vapi "github.com/vertica/vertica-kubernetes/api/v1beta1"
 	"github.com/vertica/vertica-kubernetes/pkg/controllers"
 	verrors "github.com/vertica/vertica-kubernetes/pkg/errors"
+	"github.com/vertica/vertica-kubernetes/pkg/meta"
 )
 
 // EventTriggerReconciler reconciles a EventTrigger object
@@ -77,6 +78,12 @@ func (r *EventTriggerReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
+	if meta.IsPauseAnnotationSet(et.Annotations) {
+		log.Info(fmt.Sprintf("The pause annotation %s is set. Suspending the iteration", meta.PauseOperatorAnnotation),
+			"result", ctrl.Result{}, "err", nil)
+		return ctrl.Result{}, nil
+	}
+
 	// Iterate over each actor
 	actors := r.constructActors(et, log)
 	var res ctrl.Result

pkg/controllers/et/eventtrigger_controller_test.go

@@ -22,6 +22,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	vapi "github.com/vertica/vertica-kubernetes/api/v1beta1"
+	"github.com/vertica/vertica-kubernetes/pkg/meta"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -42,4 +43,12 @@ var _ = Describe("eventtrigger_controller", func() {
 		et := vapi.MakeET()
 		Expect(etRec.Reconcile(ctx, ctrl.Request{NamespacedName: et.ExtractNamespacedName()})).Should(Equal(ctrl.Result{}))
 	})
+
+	It("should suspend the reconcile if pause annotation is set", func() {
+		et := vapi.MakeET()
+		et.Annotations = map[string]string{meta.PauseOperatorAnnotation: "1"}
+		Expect(k8sClient.Create(ctx, et)).Should(Succeed())
+		defer func() { Expect(k8sClient.Delete(ctx, et)).Should(Succeed()) }()
+		Expect(etRec.Reconcile(ctx, ctrl.Request{NamespacedName: et.ExtractNamespacedName()})).Should(Equal(ctrl.Result{}))
+	})
 })

pkg/controllers/vas/verticaautoscaler_controller.go

@@ -30,6 +30,7 @@ import (
 	vapi "github.com/vertica/vertica-kubernetes/api/v1beta1"
 	"github.com/vertica/vertica-kubernetes/pkg/controllers"
 	verrors "github.com/vertica/vertica-kubernetes/pkg/errors"
+	"github.com/vertica/vertica-kubernetes/pkg/meta"
 )
 
 // VerticaAutoscalerReconciler reconciles a VerticaAutoscaler object
@@ -67,6 +68,12 @@ func (r *VerticaAutoscalerReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		return ctrl.Result{}, err
 	}
 
+	if meta.IsPauseAnnotationSet(vas.Annotations) {
+		log.Info(fmt.Sprintf("The pause annotation %s is set. Suspending the iteration", meta.PauseOperatorAnnotation),
+			"result", ctrl.Result{}, "err", nil)
+		return ctrl.Result{}, nil
+	}
+
 	// The actors that will be applied, in sequence, to reconcile a vas.
 	actors := []controllers.ReconcileActor{
 		// Sanity check to make sure the VerticaDB referenced in vas actually exists.

pkg/controllers/vdb/createdb_reconciler.go

@@ -238,7 +238,7 @@ func (c *CreateDBReconciler) genCmd(ctx context.Context, hostList []string) ([]s
 		"--sql=" + PostDBCreateSQLFile,
 		"--catalog_path=" + c.Vdb.Spec.Local.GetCatalogPath(),
 		"--database", c.Vdb.Spec.DBName,
-		"--force-cleanup-on-failure",
+		"--force-removal-at-creation",
 		"--noprompt",
 		"--license", licPath,
 		"--depot-path=" + c.Vdb.Spec.Local.DepotPath,

pkg/controllers/vdb/verticadb_controller.go

@@ -36,6 +36,7 @@ import (
 	"github.com/vertica/vertica-kubernetes/pkg/controllers"
 	verrors "github.com/vertica/vertica-kubernetes/pkg/errors"
 	"github.com/vertica/vertica-kubernetes/pkg/events"
+	"github.com/vertica/vertica-kubernetes/pkg/meta"
 	"github.com/vertica/vertica-kubernetes/pkg/metrics"
 	"github.com/vertica/vertica-kubernetes/pkg/names"
 	"github.com/vertica/vertica-kubernetes/pkg/opcfg"
@@ -97,6 +98,12 @@ func (r *VerticaDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, err
 	}
 
+	if meta.IsPauseAnnotationSet(vdb.Annotations) {
+		log.Info(fmt.Sprintf("The pause annotation %s is set. Suspending the iteration", meta.PauseOperatorAnnotation),
+			"result", ctrl.Result{}, "err", nil)
+		return ctrl.Result{}, nil
+	}
+
 	passwd, err := r.GetSuperuserPassword(ctx, vdb, log)
 	if err != nil {
 		return ctrl.Result{}, err

pkg/meta/annotations.go

@@ -15,11 +15,29 @@
 
 package meta
 
+import "strconv"
+
 const (
 	// Annotations that we set in each of the pod.  These are set by the
 	// AnnotateAndLabelPodReconciler.  They are available in the pod with the
 	// downwardAPI so they can be picked up by the Vertica data collector (DC).
 	KubernetesVersionAnnotation   = "kubernetes.io/version"   // Version of the k8s server
 	KubernetesGitCommitAnnotation = "kubernetes.io/gitcommit" // Git commit of the k8s server
 	KubernetesBuildDateAnnotation = "kubernetes.io/buildDate" // Build date of the k8s server
+
+	// If this label is on any CR, the operator will skip processing. This can
+	// be used to avoid getting in an infinity error-retry loop. Or, if you know
+	// no additional work will ever exist for an object. Just set this to a
+	// true|ON|1 value.
+	PauseOperatorAnnotation = "vertica.com/pause"
 )
+
+// IsPauseAnnotationSet will check the annotations for a special value that will
+// pause the operator for the CR.
+func IsPauseAnnotationSet(annotations map[string]string) bool {
+	if val, ok := annotations[PauseOperatorAnnotation]; ok {
+		varAsBool, _ := strconv.ParseBool(val)
+		return varAsBool
+	}
+	return false
+}

pkg/meta/annotations_test.go

@@ -0,0 +1,45 @@
+/*
+ (c) Copyright [2021-2023] Open Text.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package meta
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestNames(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	RunSpecs(t, "annotations Suite")
+}
+
+var _ = Describe("annotations", func() {
+	It("pause operator annotation should take different boolean values", func() {
+		Ω(IsPauseAnnotationSet(nil)).Should(BeFalse())
+		ann := map[string]string{}
+		Ω(IsPauseAnnotationSet(ann)).Should(BeFalse())
+		ann[PauseOperatorAnnotation] = "true"
+		Ω(IsPauseAnnotationSet(ann)).Should(BeTrue())
+		ann[PauseOperatorAnnotation] = "1"
+		Ω(IsPauseAnnotationSet(ann)).Should(BeTrue())
+		ann[PauseOperatorAnnotation] = "OFF"
+		Ω(IsPauseAnnotationSet(ann)).Should(BeFalse())
+		ann[PauseOperatorAnnotation] = "not a bool"
+		Ω(IsPauseAnnotationSet(ann)).Should(BeFalse())
+	})
+})

tests/e2e-leg-3/labels-annotations/35-suspend-reconcile.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2023] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: vertica.com/v1beta1
+kind: VerticaDB
+metadata:
+  name: vdb-label-ant
+  annotations:
+    vertica.com/pause: "true"

tests/e2e-leg-3/labels-annotations/40-wait-for-steadystate.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2023] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n $NAMESPACE -t 360"

tests/e2e-leg-3/labels-annotations/45-delete-pod.yaml

@@ -0,0 +1,22 @@
+# (c) Copyright [2021-2023] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Delete a pod. The pause annotation that we set will have prevent the operator
+# from restarting this pod.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: v1
+    kind: Pod
+    name: vdb-label-ant-cluster1-0

tests/e2e-leg-3/labels-annotations/50-assert.yaml

@@ -0,0 +1,21 @@
+# (c) Copyright [2021-2023] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vdb-label-ant-cluster1-0
+status:
+  containerStatuses:
+  - name: server
+    ready: false

tests/e2e-leg-3/labels-annotations/50-wait-for-steadystate.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2023] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n $NAMESPACE -t 360"