Add Prometheus metrics backend

README.md

@@ -14,11 +14,18 @@ go get github.com/buildkite/buildkite-metrics
 
 ### Backends
 
-By default metrics will be submitted to CloudWatch but the backend can be switched to StatsD using the command-line argument `-backend statsd`. The StatsD backend supports the following arguments
+By default metrics will be submitted to CloudWatch but the backend can be switched to StatsD or Prometheus using the command-line argument `-backend statsd` or `-backend prometheus` respectively.
+
+The StatsD backend supports the following arguments
 
 * `-statsd-host HOST`: The StatsD host and port (defaults to `127.0.0.1:8125`).
 * `-statsd-tags`: Some StatsD servers like the agent provided by DataDog support tags. If specified, metrics will be tagged by `queue` and `pipeline` otherwise metrics will include the queue/pipeline name in the metric. Only enable this option if you know your StatsD server supports tags.
 
+The Prometheus backend supports the following arguments
+
+* `-prometheus-addr`: The local address to listen on (defaults to `:8080`).
+* `-prometheus-path`: The path under `prometheus-addr` to expose metrics on (defaults to `/metrics`).
+
 ## Development
 
 You can build and run the binary tool locally with golang installed:

backend/prometheus.go

@@ -0,0 +1,101 @@
+package backend
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/buildkite/buildkite-metrics/collector"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var (
+	camel = regexp.MustCompile("(^[^A-Z0-9]*|[A-Z0-9]*)([A-Z0-9][^A-Z]+|$)")
+)
+
+type Prometheus struct {
+	totals    map[string]prometheus.Gauge
+	queues    map[string]*prometheus.GaugeVec
+	pipelines map[string]*prometheus.GaugeVec
+}
+
+func NewPrometheusBackend(path, addr string) *Prometheus {
+	go func() {
+		http.Handle(path, promhttp.Handler())
+		log.Fatal(http.ListenAndServe(addr, nil))
+	}()
+
+	return newPrometheus()
+}
+
+func newPrometheus() *Prometheus {
+	return &Prometheus{
+		totals:    make(map[string]prometheus.Gauge),
+		queues:    make(map[string]*prometheus.GaugeVec),
+		pipelines: make(map[string]*prometheus.GaugeVec),
+	}
+}
+
+func (p *Prometheus) Collect(r *collector.Result) error {
+	for name, value := range r.Totals {
+		gauge, ok := p.totals[name]
+		if !ok {
+			gauge = prometheus.NewGauge(prometheus.GaugeOpts{
+				Name: fmt.Sprintf("buildkite_total_%s", camelToUnderscore(name)),
+				Help: fmt.Sprintf("Buildkite Total: %s", name),
+			})
+			prometheus.MustRegister(gauge)
+			p.totals[name] = gauge
+		}
+		gauge.Set(float64(value))
+	}
+
+	for queue, counts := range r.Queues {
+		for name, value := range counts {
+			gauge, ok := p.queues[name]
+			if !ok {
+				gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+					Name: fmt.Sprintf("buildkite_queues_%s", camelToUnderscore(name)),
+					Help: fmt.Sprintf("Buildkite Queues: %s", name),
+				}, []string{"queue"})
+				prometheus.MustRegister(gauge)
+				p.queues[name] = gauge
+			}
+			gauge.WithLabelValues(queue).Set(float64(value))
+		}
+	}
+
+	for pipeline, counts := range r.Pipelines {
+		for name, value := range counts {
+			gauge, ok := p.pipelines[name]
+			if !ok {
+				gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+					Name: fmt.Sprintf("buildkite_pipelines_%s", camelToUnderscore(name)),
+					Help: fmt.Sprintf("Buildkite Pipelines: %s", name),
+				}, []string{"pipeline"})
+				prometheus.MustRegister(gauge)
+				p.pipelines[name] = gauge
+			}
+			gauge.WithLabelValues(pipeline).Set(float64(value))
+		}
+	}
+
+	return nil
+}
+
+func camelToUnderscore(s string) string {
+	var a []string
+	for _, sub := range camel.FindAllStringSubmatch(s, -1) {
+		if sub[1] != "" {
+			a = append(a, sub[1])
+		}
+		if sub[2] != "" {
+			a = append(a, sub[2])
+		}
+	}
+	return strings.ToLower(strings.Join(a, "_"))
+}

backend/prometheus_test.go

@@ -0,0 +1,193 @@
+package backend
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/buildkite/buildkite-metrics/collector"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+const (
+	wantHaveFmt        = "want %v, have %v"
+	runningBuildsCount = iota
+	scheduledBuildsCount
+	runningJobsCount
+	scheduledJobsCount
+	unfinishedJobsCount
+	idleAgentCount
+	busyAgentCount
+	totalAgentCount
+)
+
+func newTestResult(t *testing.T) *collector.Result {
+	t.Helper()
+	pipelines := map[string]int{
+		"RunningBuildsCount":   runningBuildsCount,
+		"ScheduledBuildsCount": scheduledBuildsCount,
+		"RunningJobsCount":     runningJobsCount,
+		"ScheduledJobsCount":   scheduledJobsCount,
+		"UnfinishedJobsCount":  unfinishedJobsCount,
+	}
+
+	totals := make(map[string]int)
+	for k, v := range pipelines {
+		totals[k] = v
+	}
+	totals["IdleAgentCount"] = idleAgentCount
+	totals["BusyAgentCount"] = busyAgentCount
+	totals["TotalAgentCount"] = totalAgentCount
+
+	res := &collector.Result{
+		Totals: totals,
+		Queues: map[string]map[string]int{
+			"default": totals,
+			"deploy":  totals,
+		},
+		Pipelines: map[string]map[string]int{
+			"pipeline1": pipelines,
+			"pipeline2": pipelines,
+		},
+	}
+	return res
+}
+
+func gatherMetrics(t *testing.T) map[string]*dto.MetricFamily {
+	t.Helper()
+
+	oldRegisterer := prometheus.DefaultRegisterer
+	defer func() {
+		prometheus.DefaultRegisterer = oldRegisterer
+	}()
+	r := prometheus.NewRegistry()
+	prometheus.DefaultRegisterer = r
+
+	p := newPrometheus()
+	p.Collect(newTestResult(t))
+
+	if mfs, err := r.Gather(); err != nil {
+		t.Fatal(err)
+		return nil
+	} else {
+		mfsm := make(map[string]*dto.MetricFamily)
+		for _, mf := range mfs {
+			mfsm[*mf.Name] = mf
+		}
+		return mfsm
+	}
+}
+
+func TestCollect(t *testing.T) {
+	mfs := gatherMetrics(t)
+
+	if want, have := 21, len(mfs); want != have {
+		t.Errorf("wanted %d Prometheus metrics, have: %d", want, have)
+	}
+
+	tcs := []struct {
+		Group      string
+		PromName   string
+		PromHelp   string
+		PromLabels []string
+		PromValue  float64
+		PromType   dto.MetricType
+	}{
+		{
+			"Total",
+			"buildkite_total_running_jobs_count",
+			"Buildkite Total: RunningJobsCount",
+			[]string{},
+			runningJobsCount,
+			dto.MetricType_GAUGE,
+		},
+		{
+			"Total",
+			"buildkite_total_scheduled_jobs_count",
+			"Buildkite Total: ScheduledJobsCount",
+			[]string{},
+			scheduledJobsCount,
+			dto.MetricType_GAUGE,
+		},
+		{
+			"Queues",
+			"buildkite_queues_scheduled_builds_count",
+			"Buildkite Queues: ScheduledBuildsCount",
+			[]string{"default", "deploy"},
+			scheduledBuildsCount,
+			dto.MetricType_GAUGE,
+		},
+		{
+			"Queues",
+			"buildkite_queues_idle_agent_count",
+			"Buildkite Queues: IdleAgentCount",
+			[]string{"default", "deploy"},
+			idleAgentCount,
+			dto.MetricType_GAUGE,
+		},
+		{
+			"Pipelines",
+			"buildkite_pipelines_running_builds_count",
+			"Buildkite Pipelines: RunningBuildsCount",
+			[]string{"pipeline1", "pipeline2"},
+			runningBuildsCount,
+			dto.MetricType_GAUGE,
+		},
+		{
+			"Pipelines",
+			"buildkite_pipelines_unfinished_jobs_count",
+			"Buildkite Pipelines: UnfinishedJobsCount",
+			[]string{"pipeline1", "pipeline2"},
+			unfinishedJobsCount,
+			dto.MetricType_GAUGE,
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(fmt.Sprintf("%s/%s", tc.Group, tc.PromName), func(t *testing.T) {
+			mf, ok := mfs[tc.PromName]
+			if !ok {
+				t.Errorf("no metric found for name %s", tc.PromName)
+			}
+
+			if want, have := tc.PromHelp, mf.GetHelp(); want != have {
+				t.Errorf(wantHaveFmt, want, have)
+			}
+
+			if want, have := tc.PromType, mf.GetType(); want != have {
+				t.Errorf(wantHaveFmt, want, have)
+			}
+
+			ms := mf.GetMetric()
+			for i, m := range ms {
+				if want, have := tc.PromValue, m.GetGauge().GetValue(); want != have {
+					t.Errorf(wantHaveFmt, want, have)
+				}
+
+				if len(tc.PromLabels) > 0 {
+					if want, have := tc.PromLabels[i], m.Label[0].GetValue(); want != have {
+						t.Errorf(wantHaveFmt, want, have)
+					}
+				}
+			}
+
+		})
+	}
+}
+
+func TestCamelToUnderscore(t *testing.T) {
+	tcs := []struct {
+		Camel      string
+		Underscore string
+	}{
+		{"TotalAgentCount", "total_agent_count"},
+		{"Total@#4JobsCount", "total@#4_jobs_count"},
+		{"BuildkiteQueuesIdleAgentCount1_11", "buildkite_queues_idle_agent_count1_11"},
+	}
+
+	for _, tc := range tcs {
+		if want, have := tc.Underscore, camelToUnderscore(tc.Camel); want != have {
+			t.Errorf(wantHaveFmt, want, have)
+		}
+	}
+}

main.go

@@ -32,9 +32,11 @@ func main() {
 		apiEndpoint = flag.String("api-endpoint", "", "A custom buildkite api endpoint")
 
 		// backend config
-		backendOpt = flag.String("backend", "cloudwatch", "Specify the backend to send metrics to: cloudwatch, statsd")
-		statsdHost = flag.String("statsd-host", "127.0.0.1:8125", "Specify the StatsD server")
-		statsdTags = flag.Bool("statsd-tags", false, "Whether your StatsD server supports tagging like Datadog")
+		backendOpt     = flag.String("backend", "cloudwatch", "Specify the backend to use: cloudwatch, statsd, prometheus")
+		statsdHost     = flag.String("statsd-host", "127.0.0.1:8125", "Specify the StatsD server")
+		statsdTags     = flag.Bool("statsd-tags", false, "Whether your StatsD server supports tagging like Datadog")
+		prometheusAddr = flag.String("prometheus-addr", ":8080", "Prometheus metrics transport bind address")
+		prometheusPath = flag.String("prometheus-path", "/metrics", "Prometheus metrics transport path")
 
 		// filters
 		queue = flag.String("queue", "", "Only include a specific queue")
@@ -57,18 +59,20 @@ func main() {
 		os.Exit(1)
 	}
 
-	lowerBackendOpt := strings.ToLower(*backendOpt)
-	if lowerBackendOpt == "cloudwatch" {
+	switch strings.ToLower(*backendOpt) {
+	case "cloudwatch":
 		bk = backend.NewCloudWatchBackend()
-	} else if lowerBackendOpt == "statsd" {
+	case "statsd":
 		var err error
 		bk, err = backend.NewStatsDBackend(*statsdHost, *statsdTags)
 		if err != nil {
 			fmt.Printf("Error starting StatsD, err: %v\n", err)
 			os.Exit(1)
 		}
-	} else {
-		fmt.Println("Must provide a supported backend: cloudwatch, statsd")
+	case "prometheus":
+		bk = backend.NewPrometheusBackend(*prometheusPath, *prometheusAddr)
+	default:
+		fmt.Println("Must provide a supported backend: cloudwatch, statsd, prometheus")
 		os.Exit(1)
 	}