From 291b7f959af81f94d553b7e464457db4891f4957 Mon Sep 17 00:00:00 2001 From: spilchen Date: Wed, 24 May 2023 09:01:05 -0300 Subject: [PATCH] Annotation to suspend reconcile on a CR (#404) This adds the ability to suspend all reconcile activity against a custom resource. This aids in debugging because the operator uses an infinity retry model (i.e. keep retrying the reconcile and use an exponential back-off for repeated failures). This works great unless you want to dig into a specific behaviour that you are seeing. If any of the CRs (VerticaDB, VerticaAutoscaler, EventTrigger) have the `vertica.com/pause` annotation set to `true`, then no further reconcile will get scheduled for the CR. This also swaps the create DB parameter --force-cleanup-on-failure with --force-removal-at-creation. It has the same affect, but if you have the operator paused you can see the left over log files from a failed create. --- changes/unreleased/Added-20230523-142408.yaml | 5 +++ pkg/controllers/et/eventtrigger_controller.go | 7 +++ .../et/eventtrigger_controller_test.go | 9 ++++ .../vas/verticaautoscaler_controller.go | 7 +++ pkg/controllers/vdb/createdb_reconciler.go | 2 +- pkg/controllers/vdb/verticadb_controller.go | 7 +++ pkg/meta/annotations.go | 18 ++++++++ pkg/meta/annotations_test.go | 45 +++++++++++++++++++ .../35-suspend-reconcile.yaml | 19 ++++++++ .../40-wait-for-steadystate.yaml | 17 +++++++ .../labels-annotations/45-delete-pod.yaml | 22 +++++++++ .../labels-annotations/50-assert.yaml | 21 +++++++++ .../50-wait-for-steadystate.yaml | 17 +++++++ 13 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 changes/unreleased/Added-20230523-142408.yaml create mode 100644 pkg/meta/annotations_test.go create mode 100644 tests/e2e-leg-3/labels-annotations/35-suspend-reconcile.yaml create mode 100644 tests/e2e-leg-3/labels-annotations/40-wait-for-steadystate.yaml create mode 100644 tests/e2e-leg-3/labels-annotations/45-delete-pod.yaml create mode 100644 tests/e2e-leg-3/labels-annotations/50-assert.yaml create mode 100644 tests/e2e-leg-3/labels-annotations/50-wait-for-steadystate.yaml diff --git a/changes/unreleased/Added-20230523-142408.yaml b/changes/unreleased/Added-20230523-142408.yaml new file mode 100644 index 000000000..685d28a12 --- /dev/null +++ b/changes/unreleased/Added-20230523-142408.yaml @@ -0,0 +1,5 @@ +kind: Added +body: Ability to pause the reconciler for individual CRs +time: 2023-05-23T14:24:08.039581226-03:00 +custom: + Issue: "404" diff --git a/pkg/controllers/et/eventtrigger_controller.go b/pkg/controllers/et/eventtrigger_controller.go index 792d90fee..5ff74542f 100644 --- a/pkg/controllers/et/eventtrigger_controller.go +++ b/pkg/controllers/et/eventtrigger_controller.go @@ -38,6 +38,7 @@ import ( vapi "github.com/vertica/vertica-kubernetes/api/v1beta1" "github.com/vertica/vertica-kubernetes/pkg/controllers" verrors "github.com/vertica/vertica-kubernetes/pkg/errors" + "github.com/vertica/vertica-kubernetes/pkg/meta" ) // EventTriggerReconciler reconciles a EventTrigger object @@ -77,6 +78,12 @@ func (r *EventTriggerReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } + if meta.IsPauseAnnotationSet(et.Annotations) { + log.Info(fmt.Sprintf("The pause annotation %s is set. Suspending the iteration", meta.PauseOperatorAnnotation), + "result", ctrl.Result{}, "err", nil) + return ctrl.Result{}, nil + } + // Iterate over each actor actors := r.constructActors(et, log) var res ctrl.Result diff --git a/pkg/controllers/et/eventtrigger_controller_test.go b/pkg/controllers/et/eventtrigger_controller_test.go index 3ba6bc58e..006f4be88 100644 --- a/pkg/controllers/et/eventtrigger_controller_test.go +++ b/pkg/controllers/et/eventtrigger_controller_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" vapi "github.com/vertica/vertica-kubernetes/api/v1beta1" + "github.com/vertica/vertica-kubernetes/pkg/meta" ctrl "sigs.k8s.io/controller-runtime" ) @@ -42,4 +43,12 @@ var _ = Describe("eventtrigger_controller", func() { et := vapi.MakeET() Expect(etRec.Reconcile(ctx, ctrl.Request{NamespacedName: et.ExtractNamespacedName()})).Should(Equal(ctrl.Result{})) }) + + It("should suspend the reconcile if pause annotation is set", func() { + et := vapi.MakeET() + et.Annotations = map[string]string{meta.PauseOperatorAnnotation: "1"} + Expect(k8sClient.Create(ctx, et)).Should(Succeed()) + defer func() { Expect(k8sClient.Delete(ctx, et)).Should(Succeed()) }() + Expect(etRec.Reconcile(ctx, ctrl.Request{NamespacedName: et.ExtractNamespacedName()})).Should(Equal(ctrl.Result{})) + }) }) diff --git a/pkg/controllers/vas/verticaautoscaler_controller.go b/pkg/controllers/vas/verticaautoscaler_controller.go index 669ca8b71..1f84d58e5 100644 --- a/pkg/controllers/vas/verticaautoscaler_controller.go +++ b/pkg/controllers/vas/verticaautoscaler_controller.go @@ -30,6 +30,7 @@ import ( vapi "github.com/vertica/vertica-kubernetes/api/v1beta1" "github.com/vertica/vertica-kubernetes/pkg/controllers" verrors "github.com/vertica/vertica-kubernetes/pkg/errors" + "github.com/vertica/vertica-kubernetes/pkg/meta" ) // VerticaAutoscalerReconciler reconciles a VerticaAutoscaler object @@ -67,6 +68,12 @@ func (r *VerticaAutoscalerReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, err } + if meta.IsPauseAnnotationSet(vas.Annotations) { + log.Info(fmt.Sprintf("The pause annotation %s is set. Suspending the iteration", meta.PauseOperatorAnnotation), + "result", ctrl.Result{}, "err", nil) + return ctrl.Result{}, nil + } + // The actors that will be applied, in sequence, to reconcile a vas. actors := []controllers.ReconcileActor{ // Sanity check to make sure the VerticaDB referenced in vas actually exists. diff --git a/pkg/controllers/vdb/createdb_reconciler.go b/pkg/controllers/vdb/createdb_reconciler.go index 637684f04..47d8d8da3 100644 --- a/pkg/controllers/vdb/createdb_reconciler.go +++ b/pkg/controllers/vdb/createdb_reconciler.go @@ -238,7 +238,7 @@ func (c *CreateDBReconciler) genCmd(ctx context.Context, hostList []string) ([]s "--sql=" + PostDBCreateSQLFile, "--catalog_path=" + c.Vdb.Spec.Local.GetCatalogPath(), "--database", c.Vdb.Spec.DBName, - "--force-cleanup-on-failure", + "--force-removal-at-creation", "--noprompt", "--license", licPath, "--depot-path=" + c.Vdb.Spec.Local.DepotPath, diff --git a/pkg/controllers/vdb/verticadb_controller.go b/pkg/controllers/vdb/verticadb_controller.go index 7e9ef4307..fc4c2b3a0 100644 --- a/pkg/controllers/vdb/verticadb_controller.go +++ b/pkg/controllers/vdb/verticadb_controller.go @@ -36,6 +36,7 @@ import ( "github.com/vertica/vertica-kubernetes/pkg/controllers" verrors "github.com/vertica/vertica-kubernetes/pkg/errors" "github.com/vertica/vertica-kubernetes/pkg/events" + "github.com/vertica/vertica-kubernetes/pkg/meta" "github.com/vertica/vertica-kubernetes/pkg/metrics" "github.com/vertica/vertica-kubernetes/pkg/names" "github.com/vertica/vertica-kubernetes/pkg/opcfg" @@ -97,6 +98,12 @@ func (r *VerticaDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, err } + if meta.IsPauseAnnotationSet(vdb.Annotations) { + log.Info(fmt.Sprintf("The pause annotation %s is set. Suspending the iteration", meta.PauseOperatorAnnotation), + "result", ctrl.Result{}, "err", nil) + return ctrl.Result{}, nil + } + passwd, err := r.GetSuperuserPassword(ctx, vdb, log) if err != nil { return ctrl.Result{}, err diff --git a/pkg/meta/annotations.go b/pkg/meta/annotations.go index fe643c586..d518deed4 100644 --- a/pkg/meta/annotations.go +++ b/pkg/meta/annotations.go @@ -15,6 +15,8 @@ package meta +import "strconv" + const ( // Annotations that we set in each of the pod. These are set by the // AnnotateAndLabelPodReconciler. They are available in the pod with the @@ -22,4 +24,20 @@ const ( KubernetesVersionAnnotation = "kubernetes.io/version" // Version of the k8s server KubernetesGitCommitAnnotation = "kubernetes.io/gitcommit" // Git commit of the k8s server KubernetesBuildDateAnnotation = "kubernetes.io/buildDate" // Build date of the k8s server + + // If this label is on any CR, the operator will skip processing. This can + // be used to avoid getting in an infinity error-retry loop. Or, if you know + // no additional work will ever exist for an object. Just set this to a + // true|ON|1 value. + PauseOperatorAnnotation = "vertica.com/pause" ) + +// IsPauseAnnotationSet will check the annotations for a special value that will +// pause the operator for the CR. +func IsPauseAnnotationSet(annotations map[string]string) bool { + if val, ok := annotations[PauseOperatorAnnotation]; ok { + varAsBool, _ := strconv.ParseBool(val) + return varAsBool + } + return false +} diff --git a/pkg/meta/annotations_test.go b/pkg/meta/annotations_test.go new file mode 100644 index 000000000..e4d7192e3 --- /dev/null +++ b/pkg/meta/annotations_test.go @@ -0,0 +1,45 @@ +/* + (c) Copyright [2021-2023] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package meta + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestNames(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "annotations Suite") +} + +var _ = Describe("annotations", func() { + It("pause operator annotation should take different boolean values", func() { + Ω(IsPauseAnnotationSet(nil)).Should(BeFalse()) + ann := map[string]string{} + Ω(IsPauseAnnotationSet(ann)).Should(BeFalse()) + ann[PauseOperatorAnnotation] = "true" + Ω(IsPauseAnnotationSet(ann)).Should(BeTrue()) + ann[PauseOperatorAnnotation] = "1" + Ω(IsPauseAnnotationSet(ann)).Should(BeTrue()) + ann[PauseOperatorAnnotation] = "OFF" + Ω(IsPauseAnnotationSet(ann)).Should(BeFalse()) + ann[PauseOperatorAnnotation] = "not a bool" + Ω(IsPauseAnnotationSet(ann)).Should(BeFalse()) + }) +}) diff --git a/tests/e2e-leg-3/labels-annotations/35-suspend-reconcile.yaml b/tests/e2e-leg-3/labels-annotations/35-suspend-reconcile.yaml new file mode 100644 index 000000000..8946a2a28 --- /dev/null +++ b/tests/e2e-leg-3/labels-annotations/35-suspend-reconcile.yaml @@ -0,0 +1,19 @@ +# (c) Copyright [2021-2023] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: vdb-label-ant + annotations: + vertica.com/pause: "true" diff --git a/tests/e2e-leg-3/labels-annotations/40-wait-for-steadystate.yaml b/tests/e2e-leg-3/labels-annotations/40-wait-for-steadystate.yaml new file mode 100644 index 000000000..344cf20f2 --- /dev/null +++ b/tests/e2e-leg-3/labels-annotations/40-wait-for-steadystate.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2023] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n $NAMESPACE -t 360" diff --git a/tests/e2e-leg-3/labels-annotations/45-delete-pod.yaml b/tests/e2e-leg-3/labels-annotations/45-delete-pod.yaml new file mode 100644 index 000000000..92419307f --- /dev/null +++ b/tests/e2e-leg-3/labels-annotations/45-delete-pod.yaml @@ -0,0 +1,22 @@ +# (c) Copyright [2021-2023] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Delete a pod. The pause annotation that we set will have prevent the operator +# from restarting this pod. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: v1 + kind: Pod + name: vdb-label-ant-cluster1-0 diff --git a/tests/e2e-leg-3/labels-annotations/50-assert.yaml b/tests/e2e-leg-3/labels-annotations/50-assert.yaml new file mode 100644 index 000000000..2dab63b21 --- /dev/null +++ b/tests/e2e-leg-3/labels-annotations/50-assert.yaml @@ -0,0 +1,21 @@ +# (c) Copyright [2021-2023] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: vdb-label-ant-cluster1-0 +status: + containerStatuses: + - name: server + ready: false diff --git a/tests/e2e-leg-3/labels-annotations/50-wait-for-steadystate.yaml b/tests/e2e-leg-3/labels-annotations/50-wait-for-steadystate.yaml new file mode 100644 index 000000000..344cf20f2 --- /dev/null +++ b/tests/e2e-leg-3/labels-annotations/50-wait-for-steadystate.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2023] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n $NAMESPACE -t 360"