From e6fdd2939d0573ec6bf876317adfb74d600d9d97 Mon Sep 17 00:00:00 2001
From: Steven DeMartini <1647130+sjdemartini@users.noreply.github.com>
Date: Wed, 1 Jun 2022 19:46:17 -0700
Subject: [PATCH] Add support for additional Prometheus scrape configs (#127)

* Fix typos and remove trailing spaces in configuration doc

* Use consistent indentation in prometheus config file

* Add support for additional Prometheus scrape configs

* Reorder prometheus scrape config param in main.yml

Co-authored-by: Steven DeMartini <sjdemartini@users.noreply.github.com>
---
 docs/configuration.md                      | 26 +++++++++++++++++-----
 roles/prometheus/defaults/main.yml         |  2 ++
 roles/prometheus/templates/prometheus.yaml |  6 ++++-
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index eab9a5e5..2652493e 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -79,7 +79,7 @@ nfs_client_mounts:
 
 You may mount arbitrary cifs/samba or windows file shares with
 `samba_client_mounts`. The `username`, `password`, `options`, and
-`domain` fields are optional. 
+`domain` fields are optional.
 
 ```yaml
 samba_client_mounts:
@@ -94,7 +94,7 @@ samba_client_mounts:
   ...
 ```
 
-## JupyterHub 
+## JupyterHub
 
 ### Setting arbitrary traitlets in JupyterHub
 
@@ -111,7 +111,7 @@ jupyterhub_custom:
 ### Arbitrary additional files as configuration
 
 You may add additional files that are run at the end of JupyterHub's
-configuration via Traitlets. 
+configuration via Traitlets.
 
 ```yaml
 jupyterhub_additional_config:
@@ -135,7 +135,7 @@ idle_culler:
   cull_every: 3600 # 1 hour
 ```
 
- - `timeout` is the time that a user is inactive 
+ - `timeout` is the time that a user is inactive
  - `cull_every` is the interval to delete inactive jupyterlab instances
 
 ### Set default UI to classic jupyter notebooks
@@ -160,7 +160,7 @@ jupyterhub_qhub_options_form: false
 
 Additional services can be added to the `jupyterhub_services`
 variable. Currently this is only `<service-name>:
-<service-apikey>`. You must keep the `dask_gatway` section.
+<service-apikey>`. You must keep the `dask_gateway` section.
 
 ```yaml
 jupyterhub_services:
@@ -304,6 +304,21 @@ openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -sha256 -days 3
   -nodes
 ```
 
+## Prometheus
+
+### Adding additional scrape configs
+
+If you want to add additional jobs/targets for Prometheus to scrape and ingest, you can
+use the `prometheus_additional_scrape_configs` variable to define your own:
+
+```yaml
+prometheus_additional_scrape_configs:
+  - job_name: my_job
+    static_configs:
+      - targets:
+        - 'example.com:9100'
+```
+
 ## Backups
 
 Backups are performed in QHub HPC via [restic](https://restic.net/)
@@ -336,4 +351,3 @@ At any time you can trigger a manual backup. SSH into the master node.
 ```shell
 sudo systemctl start restic-backup.service
 ```
-
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
index 12fa4e4c..3d54faaa 100644
--- a/roles/prometheus/defaults/main.yml
+++ b/roles/prometheus/defaults/main.yml
@@ -8,6 +8,8 @@ node_exporter_version: "1.0.1"
 node_exporter_sha256: 3369b76cd2b0ba678b6d618deab320e565c3d93ccb5c2a0d5db51a53857768ae
 node_exporter_port: "9100"
 
+prometheus_additional_scrape_configs: []
+
 # role: jupyterhub
 jupyterhub_proxy_port: "15002"
 
diff --git a/roles/prometheus/templates/prometheus.yaml b/roles/prometheus/templates/prometheus.yaml
index 30cc56f1..aea45054 100644
--- a/roles/prometheus/templates/prometheus.yaml
+++ b/roles/prometheus/templates/prometheus.yaml
@@ -22,7 +22,7 @@ rule_files:
 scrape_configs:
   - job_name: 'prometheus'
     static_configs:
-    - targets:
+      - targets:
         - 'localhost:{{ prometheus_port }}'
 {% for host in groups['all'] %}
         - '{{ host }}:{{ node_exporter_port }}'
@@ -63,3 +63,7 @@ scrape_configs:
     static_configs:
       - targets:
         - 'localhost:{{ keycloak_port }}'
+
+{% if prometheus_additional_scrape_configs %}
+{{ prometheus_additional_scrape_configs | to_nice_yaml(indent=2) | indent(width=2, first=True) }}
+{% endif %}