Update monitoring mixin

Signed-off-by: Juraci Paixão Kröhling <[email protected]>
jaegertracing · Oct 21, 2021 · 4dcb887 · 4dcb887
1 parent a240917
commit 4dcb887
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 22 deletions.
diff --git a/monitoring/jaeger-mixin/README.md b/monitoring/jaeger-mixin/README.md
@@ -3,18 +3,19 @@
 The Prometheus monitoring mixin for Jaeger provides a starting point for people wanting to monitor Jaeger using Prometheus, Alertmanager, and Grafana. To use it, you'll need [`jsonnet`](https://github.com/google/go-jsonnet) and [`jb` (jsonnet-bundler)](https://github.com/jsonnet-bundler/jsonnet-bundler). They can be installed using `go get`, as follows:
 
 ```console
-$ go get github.com/google/go-jsonnet/cmd/jsonnet
-$ go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
+mkdir -p ~/bin && curl -sL https://github.com/google/go-jsonnet/releases/download/v0.17.0/go-jsonnet_0.17.0_Linux_x86_64.tar.gz | tar -C ~/bin/ -xzf -
+curl -sLo ~/bin/jb https://github.com/jsonnet-bundler/jsonnet-bundler/releases/download/v0.4.0/jb-linux-amd64
+chmod +x ~/bin/jb
 ```
 
 Your monitoring mixin can then be initialized as follows:
 
 ```console
-$ jb init
-$ jb install \
+jb init
+jb install \
   github.com/jaegertracing/jaeger/monitoring/jaeger-mixin@master \
   github.com/grafana/jsonnet-libs/grafana-builder@master \
-  github.com/coreos/kube-prometheus/jsonnet/kube-prometheus@master
+  github.com/coreos/kube-prometheus/jsonnet/kube-prometheus@main
 ```
 
 In the directory where your mixin was initialized, create a new `monitoring-setup.jsonnet`, specifying how your monitoring stack should look like: this file is yours, any customizations to Prometheus, Grafana, or Alertmanager should take place here. A simple example providing only the Jaeger dashboard for Grafana would be:
@@ -27,7 +28,7 @@ local jaegerDashboard = (import 'jaeger-mixin/mixin.libsonnet').grafanaDashboard
 The manifest files can be generated via the `jsonnet` command below. Once the command finishes, the file `manifests/dashboards-jaeger.json` should be available and can be loaded directly into Grafana.
 
 ```console
-$ jsonnet -J vendor -cm manifests/ monitoring-setup.jsonnet
+jsonnet -J vendor -cm manifests/ monitoring-setup.jsonnet
 ```
 
 An example producing the manifests for a complete monitoring stack is located in this directory, as `monitoring-setup.example.jsonnet`. The manifests include Prometheus, Grafana, and Alertmanager managed via the Prometheus Operator for Kubernetes.
@@ -37,17 +38,30 @@ local jaegerAlerts = (import 'jaeger-mixin/alerts.libsonnet').prometheusAlerts;
 local jaegerDashboard = (import 'jaeger-mixin/mixin.libsonnet').grafanaDashboards;
 
 local kp =
-  (import 'kube-prometheus/kube-prometheus.libsonnet') +
+  (import 'kube-prometheus/main.libsonnet') +
   {
-    _config+:: {
-      namespace: 'monitoring',
+    values+:: {
+      common+: {
+        namespace: 'observability',
+      },
+      grafana+: {
+        dashboards+:: {
+          'my-dashboard.json': jaegerDashboard['jaeger.json'],
+        },
+      },
     },
-    grafanaDashboards+:: {
-      'jaeger.json': jaegerDashboard['jaeger.json'],
+    exampleApplication: {
+      prometheusRuleExample: {
+        apiVersion: 'monitoring.coreos.com/v1',
+        kind: 'PrometheusRule',
+        metadata: {
+          name: 'my-prometheus-rule',
+          namespace: $.values.common.namespace,
+        },
+        spec: jaegerAlerts,
+      },
     },
-    prometheusAlerts+:: jaegerAlerts,
   };
-
 { ['00namespace-' + name + '.json']: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
 { ['0prometheus-operator-' + name + '.json']: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
 { ['node-exporter-' + name + '.json']: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
@@ -56,26 +70,27 @@ local kp =
 { ['prometheus-' + name + '.json']: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
 { ['prometheus-adapter-' + name + '.json']: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
 { ['grafana-' + name + '.json']: kp.grafana[name] for name in std.objectFields(kp.grafana) }
+{ ['my-application-' + name + '.json']: kp.exampleApplication[name] for name in std.objectFields(kp.exampleApplication) }
 ```
 
 The manifest files can be generated via `jsonnet` and passed directly to `kubectl`:
 
 ```console
-$ jsonnet -J vendor -cm manifests/ monitoring-setup.jsonnet
-$ kubectl apply -f manifests/
+jsonnet -J vendor -cm manifests/ monitoring-setup.jsonnet
+kubectl apply -f manifests/
 ```
 
 The resulting manifests will include everything that is needed to have a Prometheus, Alertmanager, and Grafana instances. Whenever a new alert rule is needed, or a new dashboard has to be defined, change your `monitoring-setup.jsonnet`, re-generate and re-apply the manifests.
 
 Make sure your Prometheus setup is properly scraping the Jaeger components, either by creating a `ServiceMonitor` (and the backing `Service` objects), or via `PodMonitor` resources, like:
 
 ```console
-$ kubectl apply -f - <<EOF
+kubectl apply -f - <<EOF
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: tracing
-  namespace: monitoring
+  namespace: observability
 spec:
   podMetricsEndpoints:
   - interval: 5s
@@ -97,6 +112,8 @@ This repository contains also a pre-built dashboard for Grafana and alert rules
 - [Dashboard](./dashboard-for-grafana.json)
 - [Alerts](./prometheus_alerts.yml)
 
+_IMPORTANT_: the metrics that are used by default by the dashboard are compatible with the components deployed as part of the production strategy, where each component is deployed individually. Some metric names differ from the ones used in the all-in-one strategy. Adjust your dashboard to reflect your scenario.
+
 ## Background
 
 * For more information about monitoring mixins, see this [design doc](https://docs.google.com/document/d/1A9xvzwqnFVSOZ5fD3blKODXfsat5fg6ZhnKu9LK3lB4/view).
diff --git a/monitoring/jaeger-mixin/dashboard-for-grafana.json b/monitoring/jaeger-mixin/dashboard-for-grafana.json
@@ -48,15 +48,15 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m]))",
+                     "expr": "sum(rate(jaeger_tracer_reporter_spans_total{result=~\"dropped|err\"}[1m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "error",
                      "refId": "A",
                      "step": 10
                   },
                   {
-                     "expr": "sum(rate(jaeger_reporter_spans[1m])) - sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m]))",
+                     "expr": "sum(rate(jaeger_tracer_reporter_spans_total[1m])) - sum(rate(jaeger_tracer_reporter_spans_total{result=~\"dropped|err\"}[1m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "success",
@@ -132,7 +132,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m])) by (namespace) / sum(rate(jaeger_reporter_spans[1m])) by (namespace)",
+                     "expr": "sum(rate(jaeger_tracer_reporter_spans_total{result=~\"dropped|err\"}[1m])) by (namespace) / sum(rate(jaeger_tracer_reporter_spans_total[1m])) by (namespace)",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{namespace}}",

diff --git a/monitoring/jaeger-mixin/dashboards.libsonnet b/monitoring/jaeger-mixin/dashboards.libsonnet
@@ -35,12 +35,12 @@ local g = (import 'grafana-builder/grafana.libsonnet') + {
         g.row('Services')
         .addPanel(
           g.panel('span creation rate') +
-          g.qpsPanelErrTotal('jaeger_reporter_spans{result=~"dropped|err"}', 'jaeger_reporter_spans') +
+          g.qpsPanelErrTotal('jaeger_tracer_reporter_spans_total{result=~"dropped|err"}', 'jaeger_tracer_reporter_spans_total') +
           g.stack
         )
         .addPanel(
           g.panel('% spans dropped') +
-          g.queryPanel('sum(rate(jaeger_reporter_spans{result=~"dropped|err"}[1m])) by (namespace) / sum(rate(jaeger_reporter_spans[1m])) by (namespace)', '{{namespace}}') +
+          g.queryPanel('sum(rate(jaeger_tracer_reporter_spans_total{result=~"dropped|err"}[1m])) by (namespace) / sum(rate(jaeger_tracer_reporter_spans_total[1m])) by (namespace)', '{{namespace}}') +
           { yaxes: g.yaxes({ format: 'percentunit', max: 1 }) } +
           g.stack
         )