mixin: Use sidecar's metric timestamp for healthcheck (thanos-io#3204)

During prometheus updates the alert was firing because the metric was initialized with a value of '0' before the first heartbeat was sent. As such, the evaluation of the alert results into actually taking just the value of time() into consideration which led to misleading information about the health of the sidecar. As the thanos_sidecar_last_heartbeat_success_time_seconds metric is effectively just a timestamp that resets on new deployments, we can simply wrap it around the timestamp() function which should return almost the same value of the metric itself with the added benefit that heartbeat resets will be ignored. This also refactors the relevant tests and drops the timeout to 4 minutes in order to ensure that we do not get hit by stale data if the sidecar takes longer to start. Signed-off-by: Markos Chandras <[email protected]>
dgrisonnet · Mar 26, 2021 · 80d6ef1 · 80d6ef1
1 parent 3016237
commit 80d6ef1
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 104 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 - [3919](https://github.com/thanos-io/thanos/pull/3919) Allow to disable automatically setting CORS headers using `--web.disable-cors` flag in each component that exposes an API.
 
 ### Fixed
+- [#3204](https://github.com/thanos-io/thanos/pull/3204) Mixin: Use sidecar's metric timestamp for healthcheck.
 
 - [#3204](https://github.com/thanos-io/thanos/pull/3204) Mixin: Use sidecar's metric timestamp for healthcheck.
 - [#3922](https://github.com/thanos-io/thanos/pull/3922) Fix panic in http logging middleware.

diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md
@@ -322,11 +322,12 @@ rules:
     severity: critical
 - alert: ThanosSidecarUnhealthy
   annotations:
-    description: Thanos Sidecar {{$labels.instance}} is unhealthy for {{$value}} seconds.
+    description: Thanos Sidecar {{$labels.instance}} is unhealthy for more than {{$value}}
+      seconds.
     runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
     summary: Thanos Sidecar is unhealthy.
   expr: |
-    time() - max by (job, instance) (thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}) >= 600
+    time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"})) >= 240
   labels:
     severity: critical
 ```

diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml
@@ -306,12 +306,12 @@ groups:
       severity: critical
   - alert: ThanosSidecarUnhealthy
     annotations:
-      description: Thanos Sidecar {{$labels.instance}} is unhealthy for {{$value}}
-        seconds.
+      description: Thanos Sidecar {{$labels.instance}} is unhealthy for more than
+        {{$value}} seconds.
       runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
       summary: Thanos Sidecar is unhealthy.
     expr: |
-      time() - max by (job, instance) (thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}) >= 600
+      time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"})) >= 240
     labels:
       severity: critical
 - name: thanos-store

diff --git a/examples/alerts/tests.yaml b/examples/alerts/tests.yaml
@@ -8,9 +8,9 @@ tests:
 - interval: 1m
   input_series:
   - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-0"}'
-    values: '5 10 43 17 11 0 0 0'
+    values: '5 10 43 17 11 _x5 0x10'
   - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-1"}'
-    values: '4 9 42 15 10 0 0 0'
+    values: '4 9 42 15 10 _x5 0x10'
   promql_expr_test:
     - expr: time()
       eval_time: 1m
@@ -22,112 +22,64 @@ tests:
       exp_samples:
         - labels: '{}'
           value: 120
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 2m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 43
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 42
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 10m
+    - expr: time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}))
+      eval_time: 5m
       exp_samples:
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 0
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 0
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 11m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 0
+          value: 60
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 0
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 10m
+          value: 60
+    - expr: time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}))
+      eval_time: 6m
       exp_samples:
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 600
+          value: 120
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 600
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 11m
+          value: 120
+    - expr: time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}))
+      eval_time: 7m
       exp_samples:
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 660
+          value: 180
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 660
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance) >= 600
-      eval_time: 12m
+          value: 180
+    - expr: time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}))
+      eval_time: 8m
       exp_samples:
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 720
+          value: 240
         - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 720
+          value: 240
   alert_rule_test:
-  - eval_time: 1m
-    alertname: ThanosSidecarUnhealthy
-  - eval_time: 2m
-    alertname: ThanosSidecarUnhealthy
-  - eval_time: 3m
-    alertname: ThanosSidecarUnhealthy
-  - eval_time: 10m
-    alertname: ThanosSidecarUnhealthy
-    exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for 600 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-1
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for 600 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-  - eval_time: 11m
-    alertname: ThanosSidecarUnhealthy
-    exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for 660 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-1
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for 660 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-  - eval_time: 12m
-    alertname: ThanosSidecarUnhealthy
-    exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for 720 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-1
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for 720 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
+    - eval_time: 1m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 2m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 3m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 5m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 8m
+      alertname: ThanosSidecarUnhealthy
+      exp_alerts:
+      - exp_labels:
+          severity: critical
+          job: thanos-sidecar
+          instance: thanos-sidecar-0
+        exp_annotations:
+          description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for more than 240 seconds.'
+          runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
+          summary: 'Thanos Sidecar is unhealthy.'
+      - exp_labels:
+          severity: critical
+          job: thanos-sidecar
+          instance: thanos-sidecar-1
+        exp_annotations:
+          description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for more than 240 seconds.'
+          runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
+          summary: 'Thanos Sidecar is unhealthy.'
+    - eval_time: 10m
+      alertname: ThanosSidecarUnhealthy
 - interval: 1m
   input_series:
   - series: 'prometheus_rule_evaluations_total{namespace="production", job="thanos-ruler", instance="thanos-ruler-0"}'
@@ -163,4 +115,4 @@ tests:
         exp_annotations:
           description: 'Thanos Rule thanos-ruler-1 did not perform any rule evaluations in the past 10 minutes.'
           runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosnoruleevaluations'
-          summary: 'Thanos Rule did not perform any rule evaluations.'
+          summary: 'Thanos Rule did not perform any rule evaluations.' 
diff --git a/mixin/alerts/sidecar.libsonnet b/mixin/alerts/sidecar.libsonnet
@@ -41,11 +41,11 @@
           {
             alert: 'ThanosSidecarUnhealthy',
             annotations: {
-              description: 'Thanos Sidecar {{$labels.instance}}%s is unhealthy for {{$value}} seconds.' % location,
+              description: 'Thanos Sidecar {{$labels.instance}}%s is unhealthy for more than {{$value}} seconds.' % location,
               summary: 'Thanos Sidecar is unhealthy.',
             },
             expr: |||
-              time() - max by (%(dimensions)s) (thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s}) >= 600
+              time() - max by (%(dimensions)s) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s})) >= 240
             ||| % thanos.sidecar,
             labels: {
               severity: 'critical',

diff --git a/mixin/runbook.md b/mixin/runbook.md
@@ -86,7 +86,7 @@
 |---|---|---|---|---|
 |ThanosSidecarPrometheusDown|Thanos Sidecar cannot connect to Prometheus|Thanos Sidecar {{$labels.instance}} cannot connect to Prometheus.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown)|
 |ThanosSidecarBucketOperationsFailed|Thanos Sidecar bucket operations are failing|Thanos Sidecar {{$labels.instance}} bucket operations are failing|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed)|
-|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.instance}} is unhealthy for {{$value}} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|
+|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.instance}} is unhealthy for more than {{$value}} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|
 
 ## thanos-store