Instrumentation and monitoring

.dockerdev/compose.yml

@@ -33,16 +33,17 @@ x-backend: &backend
     DATABASE_URL: postgres://postgres:postgres@postgres:5432
     BOOTSNAP_CACHE_DIR: /usr/local/bundle/_bootsnap
     WEBPACKER_DEV_SERVER_HOST: webpacker
-    WEB_CONCURRENCY: 1
-    HISTFILE: /usr/local/hist/.bash_history
-    PSQL_HISTFILE: /usr/local/hist/.psql_history
-    IRB_HISTFILE: /usr/local/hist/.irb_history
+    WEB_CONCURRENCY: ${WEB_CONCURRENCY:-1}
+    HISTFILE: /app/log/.bash_history
+    PSQL_HISTFILE: /app/log/.psql_history
+    EDITOR: vi
+    LOG: ${LOG:-0}
+    ANYCABLE_DEBUG: ${ANYCABLE_DEBUG:-1}
     ACTION_CABLE_ADAPTER: ${ACTION_CABLE_ADAPTER:-anycable}
     ANYCABLE_RPC_HOST: 0.0.0.0:50051
     ANYCABLE_BROADCAST_ADAPTER: redis
+    PROMETHEUS_EXPORTER_PORT: 5100
     CHROME_URL: http://chrome:3333
-    EDITOR: vi
-    LOG: ${LOG:-0}
   depends_on: &backend_depends_on
     postgres:
       condition: service_healthy
@@ -102,6 +103,9 @@ services:
       ANYCABLE_REDIS_URL: redis://redis:6379/0
       ANYCABLE_RPC_HOST: anycable:50051
       ANYCABLE_DEBUG: ${ANYCABLE_DEBUG:-1}
+      ANYCABLE_METRICS_HOST: "0.0.0.0"
+      ANYCABLE_METRICS_PORT: 5100
+      ANYCABLE_METRICS_HTTP: /metrics
     depends_on:
       redis:
         condition: service_healthy
@@ -179,6 +183,42 @@ services:
       # https://docs.browserless.io/docs/docker.html#connection-timeout
       CONNECTION_TIMEOUT: 600000
 
+  prometheus:
+    image: prom/prometheus:v2.29.1
+    volumes:
+      - ./etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+
+  grafana:
+    image: grafana/grafana:8.1.1
+    ports:
+      - "5000:3000"
+    volumes:
+      - ./etc/grafana/provisioning:/etc/grafana/provisioning
+      - ./etc/grafana/dashboards:/var/lib/grafana/dashboards
+      - grafana:/var/lib/grafana
+    user: "472"
+    depends_on:
+      prometheus:
+        condition: service_started
+      cadvisor:
+        condition: service_started
+
+  # Collects runtime metrics from Docker containers (CPU, RAM, etc)
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    ports:
+      - 8082:8080
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+
 volumes:
   postgres:
   redis:
@@ -188,3 +228,4 @@ volumes:
   history:
   assets:
   assets_builds:
+  grafana:

.gitignore

@@ -34,3 +34,4 @@ yarn-debug.log*
 
 /app/assets/builds/*
 !/app/assets/builds/.keep
+etc/k6/k6

Gemfile

@@ -20,6 +20,12 @@ gem 'cssbundling-rails'
 gem 'jsbundling-rails'
 gem 'propshaft'
 
+gem "yabeda-anycable"
+gem "yabeda-rails"
+gem "yabeda-puma-plugin"
+gem "yabeda-prometheus"
+gem "webrick" # For exporting metrics from AnyCable RPC
+
 group :development, :test do
   gem 'debug'
   gem 'rspec-rails', '~> 4.0.0'

Gemfile.lock

@@ -106,6 +106,7 @@ GEM
       reline (>= 0.2.7)
     diff-lcs (1.5.0)
     digest (3.1.0)
+    dry-initializer (3.0.4)
     erubi (1.10.0)
     ferrum (0.11)
       addressable (~> 2.5)
@@ -128,7 +129,8 @@ GEM
       reline (>= 0.3.0)
     jsbundling-rails (1.0.2)
       railties (>= 6.0.0)
-    loofah (2.15.0)
+    json (2.6.2)
+    loofah (2.9.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -156,11 +158,16 @@ GEM
       net-protocol
       timeout
     nio4r (2.5.8)
+    nokogiri (1.13.3-aarch64-linux)
+      racc (~> 1.4)
+    nokogiri (1.13.3-arm64-darwin)
+      racc (~> 1.4)
     nokogiri (1.13.3-x86_64-linux)
       racc (~> 1.4)
     parser (3.1.1.0)
       ast (~> 2.4.1)
     pg (1.3.4)
+    prometheus-client (2.1.0)
     propshaft (0.6.4)
       actionpack (>= 7.0.0)
       activesupport (>= 7.0.0)
@@ -240,11 +247,30 @@ GEM
     unparser (0.6.4)
       diff-lcs (~> 1.3)
       parser (>= 3.1.0)
+    webrick (1.7.0)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
+    yabeda (0.10.0)
+      anyway_config (>= 1.3, < 3)
+      concurrent-ruby
+      dry-initializer
+    yabeda-anycable (0.1.0)
+      anycable-core (~> 1.1)
+      yabeda (~> 0.10)
+    yabeda-prometheus (0.7.0)
+      prometheus-client (>= 0.10, < 3.0)
+      rack
+      yabeda (~> 0.10)
+    yabeda-puma-plugin (0.6.0)
+      json
+      puma
+      yabeda (~> 0.5)
+    yabeda-rails (0.7.2)
+      rails
+      yabeda (~> 0.8)
     zeitwerk (2.5.4)
 
 PLATFORMS
@@ -271,6 +297,11 @@ DEPENDENCIES
   ruby-next (>= 0.15.0)
   test-prof
   turbo-rails
+  webrick
+  yabeda-anycable
+  yabeda-prometheus
+  yabeda-puma-plugin
+  yabeda-rails
 
 RUBY VERSION
    ruby 3.1.1p18

README.md

@@ -59,6 +59,22 @@ dip rspec
 dip rspec system
 ```
 
+## Instrumentation
+
+You can also collect and visualize instrumentation metrics via Prometheus and Grafana:
+
+```sh
+# Start Prometheus, Grafana and Cadvisor
+dip up -d grafana
+```
+
+Open Grafana at [localhost:5000](http://localhost:5000/) (login `admin`, password `admin`) and navigate to Dashboards → Manage.
+
+Or just open dashboards using the direct links:
+
+- [AnyCable](http://localhost:5000/d/anycable-combined/anycable-combined)
+- [Docker containers CPU and RAM usage](http://localhost:5000/d/docker-containers/resource-consumption)
+
 ## Resources
 
 - [Ruby on Whales](https://evilmartians.com/chronicles/ruby-on-whales-docker-for-ruby-rails-development)—learn about the Docker development setup used for this application.

config/environments/development.rb

@@ -45,4 +45,6 @@
 
   # Raises error for missing translations.
   # config.action_view.raise_on_missing_translations = true
+
+  config.log_level = :warn if ENV["ANYCABLE_DEBUG"] == "0"
 end

config/initializers/metrics.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# To collect metrics from all processes
+Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new(dir: "/tmp/prometheus_direct_file_store")
+
+AnyCable.configure_server do
+  Yabeda::Prometheus::Exporter.start_metrics_server!
+end

config/puma.rb

@@ -29,7 +29,7 @@
 # Workers do not work on JRuby or Windows (both of which do not support
 # processes).
 #
-# workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+workers ENV.fetch("WEB_CONCURRENCY", 2)
 
 # Use the `preload_app!` method when specifying a `workers` number.
 # This directive tells Puma to first boot the application and load code
@@ -40,3 +40,7 @@
 
 # Allow puma to be restarted by `rails restart` command.
 plugin :tmp_restart
+
+activate_control_app
+plugin :yabeda
+plugin :yabeda_prometheus