dev docker compose: add otel collector with prom remote write

Add experimental support for testing otel collector together with Mimir. For reproducing #5576 Signed-off-by: György Krajcsovits <[email protected]>
grafana · Sep 7, 2023 · aa90b76 · aa90b76
1 parent 6e39892
commit aa90b76
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 1 deletion.
diff --git a/development/README.md b/development/README.md
@@ -19,6 +19,21 @@ This should give you a running Mimir system with Grafana available at [htttp://l
 
 The Minio console is available in most dev environments at [http://localhost:9001](http://localhost:9001), with the credentials defined in [mimir.yaml][minio-creds].
 
+## OTEL collector
+
+Experimental support for running OpenTelemetry collector in the Monolithic mode.
+
+### OTEL collector with Prometheus remote write
+
+Run the following command in `mimir-monolithic-mode/` to start an OpenTelemetry collector in addition to all services.
+The new service will accept OTEL metrics on port 4317 (gRPC) and 4318 (http).
+In addition the collector will scrape its own metrics.
+All received and scraped metrics are sent to Mimir via the Prometheus remote write protocol.
+
+```bash
+./compose-up.sh --profile otel-collector-remote-write
+```
+
 ## Configuring Mimir
 
 The Mimir configuration is available in each environment in the `/config` directory, along with configurations for other apps that run in the environment. Some deployment related configurations are also available at the top of each environment's `docker-compose.jsonnet` file.

diff --git a/development/mimir-monolithic-mode/compose-up.sh b/development/mimir-monolithic-mode/compose-up.sh
@@ -17,6 +17,25 @@ docker_compose() {
 
 SCRIPT_DIR=$(cd "$(dirname -- "$0")" && pwd)
 
+PROFILES=()
+ARGS=()
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --profile)
+            PROFILES+=("$1")
+            shift
+            PROFILES+=("$1")
+            shift
+            ;;
+        *)
+            ARGS+=("$1")
+            shift
+            ;;
+    esac
+done
+
+
 CGO_ENABLED=0 GOOS=linux go build -o "${SCRIPT_DIR}"/mimir "${SCRIPT_DIR}"/../../cmd/mimir && \
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build mimir-1 && \
-docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"
+docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml "${PROFILES[@]}" up "${ARGS[@]}"
diff --git a/development/mimir-monolithic-mode/config/otel-collector-remote-write-config.yaml b/development/mimir-monolithic-mode/config/otel-collector-remote-write-config.yaml
@@ -0,0 +1,36 @@
+# Based on information on https://opentelemetry.io/docs/collector/configuration/
+receivers:
+  # Data sources: traces, metrics, logs
+  otlp: # Allow sending data via OTLP protocol
+    protocols:
+      grpc:
+      http:
+
+  # Data sources: metrics
+  prometheus: # Scrape self
+    config:
+      scrape_configs:
+        - job_name: otel-collector-remote-write
+          scrape_interval: 5s
+          static_configs:
+            - targets: [localhost:8888]
+
+processors:
+  batch:
+
+exporters:
+  prometheusremotewrite:
+    endpoint: http://mimir-1:8001/api/v1/push
+
+extensions:
+  health_check:
+  pprof:
+  zpages:
+
+service:
+  extensions: [health_check, pprof, zpages]
+  pipelines:
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [batch]
+      exporters: [prometheusremotewrite]
diff --git a/development/mimir-monolithic-mode/docker-compose.yml b/development/mimir-monolithic-mode/docker-compose.yml
@@ -94,3 +94,18 @@ services:
     volumes:
       - ./config:/mimir/config
       - .data-mimir-2:/data:delegated
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.84.0
+    profiles:
+      - otel-collector-remote-write
+    volumes:
+      - ./config/otel-collector-remote-write-config.yaml:/etc/otel/config.yaml
+    ports: # From https://opentelemetry.io/docs/collector/getting-started/
+      - 1888:1888 # pprof extension
+      - 8888:8888 # Prometheus metrics exposed by the collector
+      - 8889:8889 # Prometheus exporter metrics
+      - 13133:13133 # health_check extension
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 55679:55679 # zpages extension