-
Notifications
You must be signed in to change notification settings - Fork 202
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Alert was only used inside prometheus blocker, so it allows to simplify the code. Signed-off-by: Jean-Philippe Evrard <[email protected]>
- Loading branch information
Showing
5 changed files
with
126 additions
and
138 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| package blockers | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| papi "github.com/prometheus/client_golang/api" | ||
| v1 "github.com/prometheus/client_golang/api/prometheus/v1" | ||
| "github.com/prometheus/common/model" | ||
| log "github.com/sirupsen/logrus" | ||
| "regexp" | ||
| "sort" | ||
| "time" | ||
| ) | ||
|
|
||
| // PrometheusBlockingChecker contains info for connecting | ||
| // to prometheus, and can give info about whether a reboot should be blocked | ||
| type PrometheusBlockingChecker struct { | ||
| PromConfig papi.Config | ||
| // regexp used to get alerts | ||
| Filter *regexp.Regexp | ||
| // bool to indicate if only firing alerts should be considered | ||
| FiringOnly bool | ||
| // bool to indicate that we're only blocking on alerts which match the filter | ||
| FilterMatchOnly bool | ||
| // storing the PromClient | ||
| PromClient papi.Client | ||
| } | ||
|
|
||
| func NewPrometheusBlockingChecker(config papi.Config, alertFilter *regexp.Regexp, firingOnly bool, filterMatchOnly bool) PrometheusBlockingChecker { | ||
| promClient, _ := papi.NewClient(config) | ||
|
|
||
| return PrometheusBlockingChecker{ | ||
| PromConfig: config, | ||
| Filter: alertFilter, | ||
| FiringOnly: firingOnly, | ||
| FilterMatchOnly: filterMatchOnly, | ||
| PromClient: promClient, | ||
| } | ||
| } | ||
|
|
||
| // IsBlocked for the prometheus will check if there are active alerts matching | ||
| // the arguments given into the PrometheusBlockingChecker which would actively | ||
| // block the reboot. | ||
| // As of today, no blocker information is shared as a return of the method, | ||
| // and the information is simply logged. | ||
| func (pb PrometheusBlockingChecker) IsBlocked() bool { | ||
| alertNames, err := pb.ActiveAlerts() | ||
| if err != nil { | ||
| log.Warnf("Reboot blocked: prometheus query error: %v", err) | ||
| return true | ||
| } | ||
| count := len(alertNames) | ||
| if count > 10 { | ||
| alertNames = append(alertNames[:10], "...") | ||
| } | ||
| if count > 0 { | ||
| log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames) | ||
| return true | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| // MetricLabel is used to give a fancier name | ||
| // than the type to the label for rebootBlockedCounter | ||
| func (pb PrometheusBlockingChecker) MetricLabel() string { | ||
| return "prometheus" | ||
| } | ||
|
|
||
| // ActiveAlerts is a method of type PromClient, it returns a list of names of active alerts | ||
| // (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query. | ||
| // filter by regexp means when the regexp finds the alert-name; the alert is excluded from the | ||
| // block-list and will NOT block rebooting. query by includeLabel means, | ||
| // if the query finds an alert, it will include it to the block-list, and it WILL block rebooting. | ||
| func (pb PrometheusBlockingChecker) ActiveAlerts() ([]string, error) { | ||
| api := v1.NewAPI(pb.PromClient) | ||
|
|
||
| // get all alerts from prometheus | ||
| value, _, err := api.Query(context.Background(), "ALERTS", time.Now()) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if value.Type() == model.ValVector { | ||
| if vector, ok := value.(model.Vector); ok { | ||
| activeAlertSet := make(map[string]bool) | ||
| for _, sample := range vector { | ||
| if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 { | ||
| if matchesRegex(pb.Filter, string(alertName), pb.FilterMatchOnly) && (!pb.FiringOnly || sample.Metric["alertstate"] == "firing") { | ||
| activeAlertSet[string(alertName)] = true | ||
| } | ||
| } | ||
| } | ||
|
|
||
| var activeAlerts []string | ||
| for activeAlert := range activeAlertSet { | ||
| activeAlerts = append(activeAlerts, activeAlert) | ||
| } | ||
| sort.Strings(activeAlerts) | ||
|
|
||
| return activeAlerts, nil | ||
| } | ||
| } | ||
|
|
||
| return nil, fmt.Errorf("unexpected value type %v", value) | ||
| } | ||
|
|
||
| func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool { | ||
| if filter == nil { | ||
| return true | ||
| } | ||
|
|
||
| return filter.MatchString(alertName) == filterMatchOnly | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters