mixin: Alert on receive not uploading recent data (#2612)

Signed-off-by: Frederic Branczyk <[email protected]>
thanos-io · May 15, 2020 · ecf7824 · ecf7824
1 parent 7349a08
commit ecf7824
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 0 deletions.
diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md
@@ -447,6 +447,14 @@ rules:
   for: 5m
   labels:
     severity: warning
+- alert: ThanosReceiveNoUpload
+  annotations:
+    message: Thanos Receive {{$labels.job}} has not uploaded latest data to object
+      storage.
+  expr: increase(thanos_shipper_uploads_total{job=~"thanos-receive.*"}[2h]) == 0
+  for: 30m
+  labels:
+    severity: warning
 ```
 
 ## Replicate

diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml
@@ -210,6 +210,14 @@ groups:
     for: 5m
     labels:
       severity: warning
+  - alert: ThanosReceiveNoUpload
+    annotations:
+      message: Thanos Receive {{$labels.job}} has not uploaded latest data to object
+        storage.
+    expr: increase(thanos_shipper_uploads_total{job=~"thanos-receive.*"}[2h]) == 0
+    for: 30m
+    labels:
+      severity: warning
 - name: thanos-sidecar.rules
   rules:
   - alert: ThanosSidecarPrometheusDown

diff --git a/mixin/thanos/alerts/receive.libsonnet b/mixin/thanos/alerts/receive.libsonnet
@@ -93,6 +93,17 @@
               severity: 'warning',
             },
           },
+          {
+            alert: 'ThanosReceiveNoUpload',
+            annotations: {
+              message: 'Thanos Receive {{$labels.job}} has not uploaded latest data to object storage.',
+            },
+            expr: 'increase(thanos_shipper_uploads_total{%(selector)s}[2h]) == 0' % thanos.receive,
+            'for': '30m',
+            labels: {
+              severity: 'warning',
+            },
+          },
         ],
       },
     ],