From 9fe2ca96014adbcf852fda64e23f0c13bb2e175f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Fri, 18 Oct 2024 22:11:35 +0200 Subject: [PATCH] Remove alert package Alert was only used inside prometheus blocker, so it allows to simplify the code. Signed-off-by: Jean-Philippe Evrard --- cmd/kured/main.go | 28 ++--- pkg/alerts/prometheus.go | 77 ------------- pkg/blockers/blockers.go | 37 ------- pkg/blockers/prometheus.go | 113 ++++++++++++++++++++ pkg/{alerts => blockers}/prometheus_test.go | 9 +- 5 files changed, 126 insertions(+), 138 deletions(-) delete mode 100644 pkg/alerts/prometheus.go create mode 100644 pkg/blockers/prometheus.go rename pkg/{alerts => blockers}/prometheus_test.go (96%) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 874883047..d83ca5ebd 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -4,10 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/kubereboot/kured/internal" - "github.com/kubereboot/kured/pkg/blockers" - "github.com/kubereboot/kured/pkg/checkers" - "github.com/prometheus/client_golang/prometheus/promhttp" "math/rand" "net/http" "net/url" @@ -19,8 +15,17 @@ import ( "time" "github.com/containrrr/shoutrrr" + "github.com/kubereboot/kured/internal" + "github.com/kubereboot/kured/pkg/blockers" + "github.com/kubereboot/kured/pkg/checkers" + "github.com/kubereboot/kured/pkg/daemonsetlock" + "github.com/kubereboot/kured/pkg/delaytick" + "github.com/kubereboot/kured/pkg/reboot" + "github.com/kubereboot/kured/pkg/taints" + "github.com/kubereboot/kured/pkg/timewindow" papi "github.com/prometheus/client_golang/api" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" flag "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" @@ -29,13 +34,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" kubectldrain "k8s.io/kubectl/pkg/drain" - - "github.com/kubereboot/kured/pkg/alerts" - "github.com/kubereboot/kured/pkg/daemonsetlock" - "github.com/kubereboot/kured/pkg/delaytick" - "github.com/kubereboot/kured/pkg/reboot" - "github.com/kubereboot/kured/pkg/taints" - "github.com/kubereboot/kured/pkg/timewindow" ) var ( @@ -609,12 +607,6 @@ func rebootAsRequired(nodeID string, rebooter reboot.Rebooter, checker checkers. preferNoScheduleTaint.Disable() } - // instantiate prometheus client - promClient, err := alerts.NewPromClient(papi.Config{Address: prometheusURL}) - if err != nil { - log.Fatal("Unable to create prometheus client: ", err) - } - source = rand.NewSource(time.Now().UnixNano()) tick = delaytick.New(source, period) for range tick { @@ -655,7 +647,7 @@ func rebootAsRequired(nodeID string, rebooter reboot.Rebooter, checker checkers. var blockCheckers []blockers.RebootBlocker if prometheusURL != "" { - blockCheckers = append(blockCheckers, blockers.PrometheusBlockingChecker{PromClient: promClient, Filter: alertFilter.Regexp, FiringOnly: alertFiringOnly, FilterMatchOnly: alertFilterMatchOnly}) + blockCheckers = append(blockCheckers, blockers.NewPrometheusBlockingChecker(papi.Config{Address: prometheusURL}, alertFilter.Regexp, alertFiringOnly, alertFilterMatchOnly)) } if podSelectors != nil { blockCheckers = append(blockCheckers, blockers.KubernetesBlockingChecker{Client: client, Nodename: nodeID, Filter: podSelectors}) diff --git a/pkg/alerts/prometheus.go b/pkg/alerts/prometheus.go deleted file mode 100644 index ea1457456..000000000 --- a/pkg/alerts/prometheus.go +++ /dev/null @@ -1,77 +0,0 @@ -package alerts - -import ( - "context" - "fmt" - "regexp" - "sort" - "time" - - papi "github.com/prometheus/client_golang/api" - v1 "github.com/prometheus/client_golang/api/prometheus/v1" - "github.com/prometheus/common/model" -) - -// PromClient is a wrapper around the Prometheus Client interface and implements the api -// This way, the PromClient can be instantiated with the configuration the Client needs, and -// the ability to use the methods the api has, like Query and so on. -type PromClient struct { - papi papi.Client - api v1.API -} - -// NewPromClient creates a new client to the Prometheus API. -// It returns an error on any problem. -func NewPromClient(conf papi.Config) (*PromClient, error) { - promClient, err := papi.NewClient(conf) - if err != nil { - return nil, err - } - client := PromClient{papi: promClient, api: v1.NewAPI(promClient)} - return &client, nil -} - -// ActiveAlerts is a method of type PromClient, it returns a list of names of active alerts -// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query. -// filter by regexp means when the regex finds the alert-name; the alert is exluded from the -// block-list and will NOT block rebooting. query by includeLabel means, -// if the query finds an alert, it will include it to the block-list and it WILL block rebooting. -func (p *PromClient) ActiveAlerts(filter *regexp.Regexp, firingOnly, filterMatchOnly bool) ([]string, error) { - - // get all alerts from prometheus - value, _, err := p.api.Query(context.Background(), "ALERTS", time.Now()) - if err != nil { - return nil, err - } - - if value.Type() == model.ValVector { - if vector, ok := value.(model.Vector); ok { - activeAlertSet := make(map[string]bool) - for _, sample := range vector { - if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 { - if matchesRegex(filter, string(alertName), filterMatchOnly) && (!firingOnly || sample.Metric["alertstate"] == "firing") { - activeAlertSet[string(alertName)] = true - } - } - } - - var activeAlerts []string - for activeAlert := range activeAlertSet { - activeAlerts = append(activeAlerts, activeAlert) - } - sort.Strings(activeAlerts) - - return activeAlerts, nil - } - } - - return nil, fmt.Errorf("Unexpected value type: %v", value) -} - -func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool { - if filter == nil { - return true - } - - return filter.MatchString(string(alertName)) == filterMatchOnly -} diff --git a/pkg/blockers/blockers.go b/pkg/blockers/blockers.go index e2dc15007..341e6062e 100644 --- a/pkg/blockers/blockers.go +++ b/pkg/blockers/blockers.go @@ -3,11 +3,9 @@ package blockers import ( "context" "fmt" - "github.com/kubereboot/kured/pkg/alerts" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "regexp" ) // RebootBlocked checks that a single block Checker @@ -27,20 +25,6 @@ type RebootBlocker interface { IsBlocked() bool } -// PrometheusBlockingChecker contains info for connecting -// to prometheus, and can give info about whether a reboot should be blocked -type PrometheusBlockingChecker struct { - // prometheusClient to make prometheus-go-client and api config available - // into the PrometheusBlockingChecker struct - PromClient *alerts.PromClient - // regexp used to get alerts - Filter *regexp.Regexp - // bool to indicate if only firing alerts should be considered - FiringOnly bool - // bool to indicate that we're only blocking on alerts which match the filter - FilterMatchOnly bool -} - // KubernetesBlockingChecker contains info for connecting // to k8s, and can give info about whether a reboot should be blocked type KubernetesBlockingChecker struct { @@ -51,27 +35,6 @@ type KubernetesBlockingChecker struct { Filter []string } -// IsBlocked for the prometheus will check if there are active alerts matching -// the arguments given into promclient which would actively block the reboot. -// As of today, no blocker information is shared as a return of the method, -// and the information is simply logged. -func (pb PrometheusBlockingChecker) IsBlocked() bool { - alertNames, err := pb.PromClient.ActiveAlerts(pb.Filter, pb.FiringOnly, pb.FilterMatchOnly) - if err != nil { - log.Warnf("Reboot blocked: prometheus query error: %v", err) - return true - } - count := len(alertNames) - if count > 10 { - alertNames = append(alertNames[:10], "...") - } - if count > 0 { - log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames) - return true - } - return false -} - // IsBlocked for the KubernetesBlockingChecker will check if a pod, for the node, is preventing // the reboot. It will warn in the logs about blocking, but does not return an error. func (kb KubernetesBlockingChecker) IsBlocked() bool { diff --git a/pkg/blockers/prometheus.go b/pkg/blockers/prometheus.go new file mode 100644 index 000000000..08f508e60 --- /dev/null +++ b/pkg/blockers/prometheus.go @@ -0,0 +1,113 @@ +package blockers + +import ( + "context" + "fmt" + papi "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + log "github.com/sirupsen/logrus" + "regexp" + "sort" + "time" +) + +// PrometheusBlockingChecker contains info for connecting +// to prometheus, and can give info about whether a reboot should be blocked +type PrometheusBlockingChecker struct { + PromConfig papi.Config + // regexp used to get alerts + Filter *regexp.Regexp + // bool to indicate if only firing alerts should be considered + FiringOnly bool + // bool to indicate that we're only blocking on alerts which match the filter + FilterMatchOnly bool + // storing the PromClient + PromClient papi.Client +} + +func NewPrometheusBlockingChecker(config papi.Config, alertFilter *regexp.Regexp, firingOnly bool, filterMatchOnly bool) PrometheusBlockingChecker { + promClient, _ := papi.NewClient(config) + + return PrometheusBlockingChecker{ + PromConfig: config, + Filter: alertFilter, + FiringOnly: firingOnly, + FilterMatchOnly: filterMatchOnly, + PromClient: promClient, + } +} + +// IsBlocked for the prometheus will check if there are active alerts matching +// the arguments given into the PrometheusBlockingChecker which would actively +// block the reboot. +// As of today, no blocker information is shared as a return of the method, +// and the information is simply logged. +func (pb PrometheusBlockingChecker) IsBlocked() bool { + alertNames, err := pb.ActiveAlerts() + if err != nil { + log.Warnf("Reboot blocked: prometheus query error: %v", err) + return true + } + count := len(alertNames) + if count > 10 { + alertNames = append(alertNames[:10], "...") + } + if count > 0 { + log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames) + return true + } + return false +} + +// MetricLabel is used to give a fancier name +// than the type to the label for rebootBlockedCounter +func (pb PrometheusBlockingChecker) MetricLabel() string { + return "prometheus" +} + +// ActiveAlerts is a method of type PromClient, it returns a list of names of active alerts +// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query. +// filter by regexp means when the regexp finds the alert-name; the alert is excluded from the +// block-list and will NOT block rebooting. query by includeLabel means, +// if the query finds an alert, it will include it to the block-list, and it WILL block rebooting. +func (pb PrometheusBlockingChecker) ActiveAlerts() ([]string, error) { + api := v1.NewAPI(pb.PromClient) + + // get all alerts from prometheus + value, _, err := api.Query(context.Background(), "ALERTS", time.Now()) + if err != nil { + return nil, err + } + + if value.Type() == model.ValVector { + if vector, ok := value.(model.Vector); ok { + activeAlertSet := make(map[string]bool) + for _, sample := range vector { + if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 { + if matchesRegex(pb.Filter, string(alertName), pb.FilterMatchOnly) && (!pb.FiringOnly || sample.Metric["alertstate"] == "firing") { + activeAlertSet[string(alertName)] = true + } + } + } + + var activeAlerts []string + for activeAlert := range activeAlertSet { + activeAlerts = append(activeAlerts, activeAlert) + } + sort.Strings(activeAlerts) + + return activeAlerts, nil + } + } + + return nil, fmt.Errorf("unexpected value type %v", value) +} + +func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool { + if filter == nil { + return true + } + + return filter.MatchString(alertName) == filterMatchOnly +} diff --git a/pkg/alerts/prometheus_test.go b/pkg/blockers/prometheus_test.go similarity index 96% rename from pkg/alerts/prometheus_test.go rename to pkg/blockers/prometheus_test.go index d2a224f34..f361aafe5 100644 --- a/pkg/alerts/prometheus_test.go +++ b/pkg/blockers/prometheus_test.go @@ -1,4 +1,4 @@ -package alerts +package blockers import ( "log" @@ -145,12 +145,9 @@ func TestActiveAlerts(t *testing.T) { regex, _ := regexp.Compile(tc.rFilter) // instantiate the prometheus client with the mockserver-address - p, err := NewPromClient(api.Config{Address: mockServer.URL}) - if err != nil { - log.Fatal(err) - } + p := NewPrometheusBlockingChecker(api.Config{Address: mockServer.URL}, regex, tc.firingOnly, tc.filterMatchOnly) - result, err := p.ActiveAlerts(regex, tc.firingOnly, tc.filterMatchOnly) + result, err := p.ActiveAlerts() if err != nil { log.Fatal(err) }