From 0edf95835adf4cc0e2816cc688e2005788bc12e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Gouteroux?= <francois.gouteroux@gmail.com>
Date: Mon, 16 May 2022 21:06:33 +0200
Subject: [PATCH] Add metric thanos alert sent by alert name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: François Gouteroux <francois.gouteroux@gmail.com>
---
 pkg/alert/alert.go      | 18 ++++++++++++++----
 pkg/alert/alert_test.go |  8 +++++++-
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go
index 68f6b3fd6dc..8826371e1ea 100644
--- a/pkg/alert/alert.go
+++ b/pkg/alert/alert.go
@@ -221,10 +221,11 @@ type Sender struct {
 	alertmanagers []*Alertmanager
 	versions      []APIVersion
 
-	sent    *prometheus.CounterVec
-	errs    *prometheus.CounterVec
-	dropped prometheus.Counter
-	latency *prometheus.HistogramVec
+	sent            *prometheus.CounterVec
+	sentByAlertName *prometheus.CounterVec
+	errs            *prometheus.CounterVec
+	dropped         prometheus.Counter
+	latency         *prometheus.HistogramVec
 }
 
 // NewSender returns a new sender. On each call to Send the entire alert batch is sent
@@ -257,6 +258,11 @@ func NewSender(
 			Help: "Total number of alerts sent by alertmanager.",
 		}, []string{"alertmanager"}),
 
+		sentByAlertName: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+			Name: "thanos_alert_sender_alerts_sent_by_alertname_total",
+			Help: "Total number of alerts sent by alertmanager and alert name.",
+		}, []string{"alertmanager", "alertname"}),
+
 		errs: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Name: "thanos_alert_sender_errors_total",
 			Help: "Total number of errors while sending alerts to alertmanager.",
@@ -352,6 +358,10 @@ func (s *Sender) Send(ctx context.Context, alerts []*notifier.Alert) {
 					s.latency.WithLabelValues(u.Host).Observe(time.Since(start).Seconds())
 					s.sent.WithLabelValues(u.Host).Add(float64(len(alerts)))
 
+					for _, alert := range alerts {
+						s.sentByAlertName.WithLabelValues(u.Host, alert.Name()).Inc()
+					}
+
 					numSuccess.Inc()
 				})
 			}(am, *u)
diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go
index cca97107e82..0e64e0fe055 100644
--- a/pkg/alert/alert_test.go
+++ b/pkg/alert/alert_test.go
@@ -135,7 +135,10 @@ func TestSenderSendsOk(t *testing.T) {
 	}
 	s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute, APIv1)})
 
-	s.Send(context.Background(), []*notifier.Alert{{}, {}})
+	s.Send(context.Background(), []*notifier.Alert{
+		{Labels: labels.FromStrings("alertname", "test")}, {
+			Labels: labels.FromStrings("alertname", "test"),
+		}})
 
 	assertSameHosts(t, poster.urls, poster.seen)
 
@@ -145,6 +148,9 @@ func TestSenderSendsOk(t *testing.T) {
 	testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[1].Host))))
 	testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[1].Host))))
 	testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.dropped)))
+
+	testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sentByAlertName.WithLabelValues(poster.urls[0].Host, "test"))))
+	testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sentByAlertName.WithLabelValues(poster.urls[1].Host, "test"))))
 }
 
 func TestSenderSendsOneFails(t *testing.T) {