From e27a9e24823863c9529aa3f4e2e1023d613179aa Mon Sep 17 00:00:00 2001 From: Sunny Date: Fri, 11 Aug 2023 14:16:58 +0000 Subject: [PATCH] Delete stale metrics on object delete The metrics helper now accepts owned finalizers to determine if an object is no longer managed by the controller and is being deleted, and deletes the metrics associated with the object. Call the metrics recording defer function in controller early to be able to record the object in deleting state. Signed-off-by: Sunny --- go.mod | 2 +- go.sum | 4 ++-- .../imageupdateautomation_controller.go | 16 +++++++--------- main.go | 3 ++- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index afa6cd8e..d119049e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/fluxcd/pkg/git v0.12.4 github.com/fluxcd/pkg/git/gogit v0.12.1 github.com/fluxcd/pkg/gittestserver v0.8.5 - github.com/fluxcd/pkg/runtime v0.41.0 + github.com/fluxcd/pkg/runtime v0.42.0 github.com/fluxcd/pkg/ssh v0.8.1 github.com/fluxcd/source-controller/api v1.0.1 github.com/go-git/go-billy/v5 v5.4.1 diff --git a/go.sum b/go.sum index 45564880..1d59ee33 100644 --- a/go.sum +++ b/go.sum @@ -95,8 +95,8 @@ github.com/fluxcd/pkg/git/gogit v0.12.1 h1:06jzHOTntYN5xCSQvyFXtLXdqoP8crLh7VYgt github.com/fluxcd/pkg/git/gogit v0.12.1/go.mod h1:Z4Ysp8VifKTvWpjJMKncJsgb2iBqHuIeK80VGjlU41Y= github.com/fluxcd/pkg/gittestserver v0.8.5 h1:EGqDF4240xPRgW1FFrQAs0Du7fZb8OGXC5qKDIqyXD8= github.com/fluxcd/pkg/gittestserver v0.8.5/go.mod h1:SyGEh+OBzFpdlTWWqv3XBkiLB42Iu+mijfIQ4hPlEZQ= -github.com/fluxcd/pkg/runtime v0.41.0 h1:hjWUwVRCKDuGEUhovWrygt/6PRry4p278yKuJNgTfv8= -github.com/fluxcd/pkg/runtime v0.41.0/go.mod h1:1GN+nxoQ7LmSsLJwjH8JW8pA27tBSO+KLH43HpywCDM= +github.com/fluxcd/pkg/runtime v0.42.0 h1:a5DQ/f90YjoHBmiXZUpnp4bDSLORjInbmqP7K11L4uY= +github.com/fluxcd/pkg/runtime v0.42.0/go.mod h1:p6A3xWVV8cKLLQW0N90GehKgGMMmbNYv+OSJ/0qB0vg= github.com/fluxcd/pkg/ssh v0.8.1 h1:v35y7Ks/+ABWce8RcnrC7psVIhf3EdCUNFJi5+tYOps= github.com/fluxcd/pkg/ssh v0.8.1/go.mod h1:M1ouDXuDG+QuhGB4JYEjCNCykNytLJGDhwKn9y4DEOE= github.com/fluxcd/pkg/version v0.2.2 h1:ZpVXECeLA5hIQMft11iLp6gN3cKcz6UNuVTQPw/bRdI= diff --git a/internal/controller/imageupdateautomation_controller.go b/internal/controller/imageupdateautomation_controller.go index 22448d5c..61d1c91a 100644 --- a/internal/controller/imageupdateautomation_controller.go +++ b/internal/controller/imageupdateautomation_controller.go @@ -110,6 +110,13 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, client.IgnoreNotFound(err) } + defer func() { + // Always record suspend, readiness and duration metrics. + r.Metrics.RecordSuspend(ctx, &auto, auto.Spec.Suspend) + r.Metrics.RecordReadiness(ctx, &auto) + r.Metrics.RecordDuration(ctx, &auto, start) + }() + // If the object is under deletion, record the readiness, and remove our finalizer. if !auto.ObjectMeta.DeletionTimestamp.IsZero() { controllerutil.RemoveFinalizer(&auto, imagev1.ImageUpdateAutomationFinalizer) @@ -131,9 +138,6 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr } } - // record suspension metrics - r.RecordSuspend(ctx, &auto, auto.Spec.Suspend) - if auto.Spec.Suspend { log.Info("ImageUpdateAutomation is suspended, skipping automation run") return ctrl.Result{}, nil @@ -141,12 +145,6 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr templateValues.AutomationObject = req.NamespacedName - defer func() { - // Always record readiness and duration metrics - r.Metrics.RecordReadiness(ctx, &auto) - r.Metrics.RecordDuration(ctx, &auto, start) - }() - // whatever else happens, we've now "seen" the reconcile // annotation if it's there if token, ok := meta.ReconcileAnnotationValue(auto.GetAnnotations()); ok { diff --git a/main.go b/main.go index 833bb967..804e4e7a 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ import ( feathelper "github.com/fluxcd/pkg/runtime/features" "github.com/fluxcd/pkg/runtime/leaderelection" "github.com/fluxcd/pkg/runtime/logger" + "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/pprof" "github.com/fluxcd/pkg/runtime/probes" sourcev1 "github.com/fluxcd/source-controller/api/v1" @@ -184,7 +185,7 @@ func main() { os.Exit(1) } - metricsH := helper.MustMakeMetrics(mgr) + metricsH := helper.NewMetrics(mgr, metrics.MustMakeRecorder(), imagev1.ImageUpdateAutomationFinalizer) ctx := ctrl.SetupSignalHandler()