Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix regression introduced in previous PR #3884

Merged
merged 1 commit into from
Mar 5, 2021
Merged

Fix regression introduced in previous PR #3884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions examples/alerts/alerts.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,12 +324,12 @@ rules:
severity: critical
- alert: ThanosSidecarUnhealthy
annotations:
description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for more
than {{ $value }} seconds.
description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy
for more than {{$value}} seconds.
runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
summary: Thanos Sidecar is unhealthy.
expr: |
time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"})) by (job,pod) >= 240
time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"})) >= 240
labels:
severity: critical
```
Expand Down
6 changes: 3 additions & 3 deletions examples/alerts/alerts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,12 +308,12 @@ groups:
severity: critical
- alert: ThanosSidecarUnhealthy
annotations:
description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for
more than {{ $value }} seconds.
description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy
for more than {{$value}} seconds.
runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
summary: Thanos Sidecar is unhealthy.
expr: |
time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"})) by (job,pod) >= 240
time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"})) >= 240
labels:
severity: critical
- name: thanos-store
Expand Down
36 changes: 18 additions & 18 deletions examples/alerts/tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ evaluation_interval: 1m
tests:
- interval: 1m
input_series:
- series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
- series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-0"}'
values: '5 10 43 17 11 _x5 0x10'
- series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
- series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-1"}'
values: '4 9 42 15 10 _x5 0x10'
promql_expr_test:
- expr: time()
Expand All @@ -22,33 +22,33 @@ tests:
exp_samples:
- labels: '{}'
value: 120
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, instance)
eval_time: 5m
exp_samples:
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
value: 60
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
value: 60
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, instance)
eval_time: 6m
exp_samples:
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
value: 120
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
value: 120
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, instance)
eval_time: 7m
exp_samples:
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
value: 180
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
value: 180
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
- expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, instance)
eval_time: 8m
exp_samples:
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
value: 240
- labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
- labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
value: 240
alert_rule_test:
- eval_time: 1m
Expand All @@ -65,17 +65,17 @@ tests:
- exp_labels:
severity: critical
job: thanos-sidecar
pod: thanos-sidecar-pod-0
instance: thanos-sidecar-0
exp_annotations:
description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-pod-0 is unhealthy for more than 240 seconds.'
description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-0 is unhealthy for more than 240 seconds.'
runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
summary: 'Thanos Sidecar is unhealthy.'
- exp_labels:
severity: critical
job: thanos-sidecar
pod: thanos-sidecar-pod-1
instance: thanos-sidecar-1
exp_annotations:
description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-pod-1 is unhealthy for more than 240 seconds.'
description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-1 is unhealthy for more than 240 seconds.'
runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
summary: 'Thanos Sidecar is unhealthy.'
- eval_time: 10m
Expand Down
4 changes: 2 additions & 2 deletions mixin/alerts/sidecar.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@
{
alert: 'ThanosSidecarUnhealthy',
annotations: {
description: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for more than {{ $value }} seconds.',
description: 'Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy for more than {{$value}} seconds.',
summary: 'Thanos Sidecar is unhealthy.',
},
expr: |||
time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s})) by (job,pod) >= 240
time() - max by (job, instance) (timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s})) >= 240
||| % thanos.sidecar,
labels: {
severity: 'critical',
Expand Down
2 changes: 1 addition & 1 deletion mixin/runbook.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@
|---|---|---|---|---|
|ThanosSidecarPrometheusDown|Thanos Sidecar cannot connect to Prometheus|Thanos Sidecar {{$labels.job}} {{$labels.instance}} cannot connect to Prometheus.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown)|
|ThanosSidecarBucketOperationsFailed|Thanos Sidecar bucket operations are failing|Thanos Sidecar {{$labels.job}} {{$labels.instance}} bucket operations are failing|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed)|
|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for more than {{ $value }} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|
|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy for more than {{$value}} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|

## thanos-store

Expand Down