Add metric thanos alert sent by alert name

Signed-off-by: François Gouteroux <[email protected]>
thanos-io · May 22, 2022 · 0edf958 · 0edf958
1 parent 9812db5
commit 0edf958
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 5 deletions.
diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go
@@ -221,10 +221,11 @@ type Sender struct {
 	alertmanagers []*Alertmanager
 	versions      []APIVersion
 
-	sent    *prometheus.CounterVec
-	errs    *prometheus.CounterVec
-	dropped prometheus.Counter
-	latency *prometheus.HistogramVec
+	sent            *prometheus.CounterVec
+	sentByAlertName *prometheus.CounterVec
+	errs            *prometheus.CounterVec
+	dropped         prometheus.Counter
+	latency         *prometheus.HistogramVec
 }
 
 // NewSender returns a new sender. On each call to Send the entire alert batch is sent
@@ -257,6 +258,11 @@ func NewSender(
 			Help: "Total number of alerts sent by alertmanager.",
 		}, []string{"alertmanager"}),
 
+		sentByAlertName: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+			Name: "thanos_alert_sender_alerts_sent_by_alertname_total",
+			Help: "Total number of alerts sent by alertmanager and alert name.",
+		}, []string{"alertmanager", "alertname"}),
+
 		errs: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Name: "thanos_alert_sender_errors_total",
 			Help: "Total number of errors while sending alerts to alertmanager.",
@@ -352,6 +358,10 @@ func (s *Sender) Send(ctx context.Context, alerts []*notifier.Alert) {
 					s.latency.WithLabelValues(u.Host).Observe(time.Since(start).Seconds())
 					s.sent.WithLabelValues(u.Host).Add(float64(len(alerts)))
 
+					for _, alert := range alerts {
+						s.sentByAlertName.WithLabelValues(u.Host, alert.Name()).Inc()
+					}
+
 					numSuccess.Inc()
 				})
 			}(am, *u)

diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go
@@ -135,7 +135,10 @@ func TestSenderSendsOk(t *testing.T) {
 	}
 	s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute, APIv1)})
 
-	s.Send(context.Background(), []*notifier.Alert{{}, {}})
+	s.Send(context.Background(), []*notifier.Alert{
+		{Labels: labels.FromStrings("alertname", "test")}, {
+			Labels: labels.FromStrings("alertname", "test"),
+		}})
 
 	assertSameHosts(t, poster.urls, poster.seen)
 
@@ -145,6 +148,9 @@ func TestSenderSendsOk(t *testing.T) {
 	testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[1].Host))))
 	testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[1].Host))))
 	testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.dropped)))
+
+	testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sentByAlertName.WithLabelValues(poster.urls[0].Host, "test"))))
+	testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sentByAlertName.WithLabelValues(poster.urls[1].Host, "test"))))
 }
 
 func TestSenderSendsOneFails(t *testing.T) {