fix and generate examples

Signed-off-by: Thibault Mange <[email protected]>
thanos-io · Jun 22, 2023 · d7793f0 · d7793f0
1 parent 93c50db
commit d7793f0
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml
@@ -274,6 +274,24 @@ groups:
     for: 3h
     labels:
       severity: critical
+  - alert: ThanosReceiveLimitsConfigReloadFailure
+    annotations:
+      description: Thanos Receive {{$labels.job}} has not been able to reload the limits configuration.
+      runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivelimitsconfigreloadfailure
+      summary: Thanos Receive has not been able to reload the limits configuration.
+    expr: sum by(job) (increase(thanos_receive_limits_config_reload_err_total{job=~".*thanos-receive.*"}[5m])) > 0
+    for: 5m
+    labels:
+      severity: warning
+  - alert: ThanosReceiveLimitsHighMetaMonitoringQueriesFailureRate
+    annotations:
+      description: Thanos Receive {{$labels.job}} is failing for {{$value | humanize}}% of meta monitoring queries.
+      runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivelimitshighmetamonitoringqueriesfailurerate
+      summary: Thanos Receive has not been able to update the number of head series.
+    expr: sum by(job) (increase(thanos_receive_metamonitoring_failed_queries_total{job=~".*thanos-receive.*"}[5m])) / 20) * 100 > 20
+    for: 5m
+    labels:
+      severity: warning
 - name: thanos-sidecar
   rules:
   - alert: ThanosSidecarBucketOperationsFailed

diff --git a/mixin/alerts/receive.libsonnet b/mixin/alerts/receive.libsonnet
@@ -163,13 +163,8 @@
               description: 'Thanos Receive {{$labels.job}}%s is failing for {{$value | humanize}}%% of meta monitoring queries.' % location,
               summary: 'Thanos Receive has not been able to update the number of head series.',
             },
-            expr: |||
-              (
-                sum by(%(dimensions)s) (increase(thanos_receive_metamonitoring_failed_queries_total{%(selector)s}[5m])) 
-              / 
-                20
-              ) * 100 > %(metaMonitoringErrorThreshold)s # Values are updated every 15s, 20 times over 5 minutes.'
-            ||| % thanos.receive,
+            // Values are updated every 15s, 20 times over 5 minutes.
+            expr: '(sum by(%(dimensions)s) (increase(thanos_receive_metamonitoring_failed_queries_total{%(selector)s}[5m])) / 20) * 100 > %(metaMonitoringErrorThreshold)s' % thanos.receive,
             'for': '5m',
             labels: {
               severity: 'warning',

diff --git a/mixin/runbook.md b/mixin/runbook.md
@@ -63,6 +63,8 @@
 |ThanosReceiveHighHashringFileRefreshFailures|Thanos Receive is failing to refresh hasring file.|Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{$value  humanize}} of attempts failed.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures)|
 |ThanosReceiveConfigReloadFailure|Thanos Receive has not been able to reload configuration.|Thanos Receive {{$labels.job}} has not been able to reload hashring configurations.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure)|
 |ThanosReceiveNoUpload|Thanos Receive has not uploaded latest data to object storage.|Thanos Receive {{$labels.instance}} has not uploaded latest data to object storage.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload)|
+|ThanosReceiveLimitsConfigReloadFailure|Thanos Receive has not been able to reload the limits configuration.|Thanos Receive {{$labels.job}} has not been able to reload the limits configuration.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivelimitsconfigreloadfailure](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivelimitsconfigreloadfailure)|
+|ThanosReceiveLimitsHighMetaMonitoringQueriesFailureRate|Thanos Receive has not been able to update the number of head series.|Thanos Receive {{$labels.job}} is failing for {{$value  humanize}}% of meta monitoring queries.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivelimitshighmetamonitoringqueriesfailurerate](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivelimitshighmetamonitoringqueriesfailurerate)|
 
 ## thanos-rule