Initial integration of DogStatsD

README.md

@@ -44,6 +44,9 @@
   - [Statistics](#statistics)
 - [Statistics](#statistics-1)
   - [Statistics options](#statistics-options)
+  - [DogStatsD](#dogstatsd)
+    - [Example](#example)
+    - [Continued example:](#continued-example)
 - [HTTP Port](#http-port)
   - [/json endpoint](#json-endpoint)
 - [Debug Port](#debug-port)
@@ -846,6 +849,57 @@ ratelimit.service.rate_limit.messaging.auth-service.over_limit.shadow_mode: 1
 
 1. `EXTRA_TAGS`: set to `"<k1:v1>,<k2:v2>"` to tag all emitted stats with the provided tags. You might want to tag build commit or release version, for example.
 
+## DogStatsD
+
+To enable dogstatsd integration set:
+
+1. `USE_DOG_STATSD`: `true` to use [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/?code-lang=go)
+
+dogstatsd also enables so called `mogrifiers` which can
+convert from traditional stats tags into a combination of stat name and tags.
+
+To enable mogrifiers, set a comma-separated list of them in `DOG_STATSD_MOGRIFIERS`.
+
+e.g. `USE_DOG_STATSD_MOGRIFIERS`: `FOO,BAR`
+
+For each mogrifier, define variables that declare the mogrification
+
+1. `DOG_STATSD_MOGRIFIERS_%s_PATTERN`: The regex pattern to match on
+2. `DOG_STATSD_MOGRIFIERS_%s_NAME`: The name of the metric to emit. Can contain variables.
+3. `DOG_STATSD_MOGRIFIERS_%s_TAGS`: Comma-separated list of tags to emit. Can contain variables.
+
+Variables within mogrifiers are strings such as `$1`, `$2`, `$3` which can be used to reference
+a match group from the regex pattern.
+
+### Example
+
+In the example below we will set mogrifier DOMAIN to adjust
+`some.original.metric.TAG` to `some.original.metric` with tag `domain:TAG`
+
+First enable a single mogrifier:
+
+1. `USE_DOG_STATSD_MOGRIFIERS`: `DOMAIN`
+
+Then, declare the rules for the `DOMAIN` modifier:
+
+1. `DOG_STATSD_MOGRIFIER_DOMAIN_PATTERN`: `^some\.original\.metric\.(.*)$`
+2. `DOG_STATSD_MOGRIFIER_DOMAIN_NAME`: `some.original.metric`
+3. `DOG_STATSD_MOGRIFIER_DOMAIN_TAGS`: `domain:$1`
+
+### Continued example:
+
+Let's also set another mogrifier which outputs the hits metrics with a domain and descriptor tag
+
+First, enable an extra mogrifier:
+
+1. `USE_DOG_STATSD_MOGRIFIERS`: `DOMAIN,HITS`
+
+Then, declare additional rules for the `DESCRIPTOR` mogrifier
+
+1. `DOG_STATSD_MOGRIFIER_HITS_PATTERN`: `^ratelimit\.service\.rate_limit\.(.*)\.(.*)\.(.*)$`
+2. `DOG_STATSD_MOGRIFIER_HITS_NAME`: `ratelimit.service.rate_limit.$3`
+3. `DOG_STATSD_MOGRIFIER_HITS_TAGS`: `domain:$1,descriptor:$2`
+
 # HTTP Port
 
 The ratelimit service listens to HTTP 1.1 (by default on port 8080) with two endpoints:

go.mod

@@ -3,6 +3,7 @@ module github.com/envoyproxy/ratelimit
 go 1.21.5
 
 require (
+	github.com/DataDog/datadog-go/v5 v5.5.0
 	github.com/alicebob/miniredis/v2 v2.31.0
 	github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874
 	github.com/coocood/freecache v1.2.4
@@ -34,6 +35,7 @@ require (
 
 require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/Microsoft/go-winio v0.5.0 // indirect
 	github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect

go.sum

@@ -5,7 +5,11 @@ cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//u
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
+github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
+github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU=
+github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
 github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
@@ -101,6 +105,7 @@ github.com/mediocregopher/radix/v3 v3.8.1 h1:rOkHflVuulFKlwsLY01/M2cM2tWCjDoETcM
 github.com/mediocregopher/radix/v3 v3.8.1/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/planetscale/vtprotobuf v0.5.1-0.20231212170721-e7d721933795 h1:pH+U6pJP0BhxqQ4njBUjOg0++WMMvv3eByWzB+oATBY=
 github.com/planetscale/vtprotobuf v0.5.1-0.20231212170721-e7d721933795/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -110,16 +115,22 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -189,9 +200,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

src/godogstats/dogstatsd_sink.go

@@ -0,0 +1,82 @@
+package godogstats
+
+import (
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/DataDog/datadog-go/v5/statsd"
+	gostats "github.com/lyft/gostats"
+)
+
+type godogStatsSink struct {
+	client *statsd.Client
+	config struct {
+		host string
+		port int
+	}
+
+	mogrifier mogrifierMap
+}
+
+// ensure that godogStatsSink implements gostats.Sink
+var _ gostats.Sink = (*godogStatsSink)(nil)
+
+type goDogStatsSinkOption func(*godogStatsSink)
+
+func WithStatsdHost(host string) goDogStatsSinkOption {
+	return func(g *godogStatsSink) {
+		g.config.host = host
+	}
+}
+
+func WithStatsdPort(port int) goDogStatsSinkOption {
+	return func(g *godogStatsSink) {
+		g.config.port = port
+	}
+}
+
+func WithMogrifier(mogrifiers map[*regexp.Regexp]func([]string) (string, []string)) goDogStatsSinkOption {
+	return func(g *godogStatsSink) {
+		g.mogrifier = mogrifiers
+	}
+}
+
+func WithMogrifierFromEnv(keys []string) goDogStatsSinkOption {
+	return func(g *godogStatsSink) {
+		mogrifier, err := newMogrifierMapFromEnv(keys)
+		if err != nil {
+			panic(err)
+		}
+		g.mogrifier = mogrifier
+	}
+}
+
+func NewSink(opts ...goDogStatsSinkOption) (*godogStatsSink, error) {
+	sink := &godogStatsSink{}
+	for _, opt := range opts {
+		opt(sink)
+	}
+	client, err := statsd.New(sink.config.host+":"+strconv.Itoa(sink.config.port), statsd.WithoutClientSideAggregation())
+	if err != nil {
+		return nil, err
+	}
+	sink.client = client
+	return sink, nil
+}
+
+func (g *godogStatsSink) FlushCounter(name string, value uint64) {
+	name, tags := g.mogrifier.mogrify(name)
+	g.client.Count(name, int64(value), tags, 1.0)
+}
+
+func (g *godogStatsSink) FlushGauge(name string, value uint64) {
+	name, tags := g.mogrifier.mogrify(name)
+	g.client.Gauge(name, float64(value), tags, 1.0)
+}
+
+func (g *godogStatsSink) FlushTimer(name string, milliseconds float64) {
+	name, tags := g.mogrifier.mogrify(name)
+	duration := time.Duration(milliseconds) * time.Millisecond
+	g.client.Timing(name, duration, tags, 1.0)
+}

src/godogstats/mogrifier_map.go

@@ -0,0 +1,107 @@
+package godogstats
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/kelseyhightower/envconfig"
+)
+
+var varFinder = regexp.MustCompile(`\$\d+`) // matches $0, $1, etc.
+
+const envPrefix = "DOG_STATSD_MOGRIFIER" // prefix for environment variables
+
+// mogrifierMap is a map of regular expressions to functions that mogrify a name and return tags
+type mogrifierMap map[*regexp.Regexp]func([]string) (string, []string)
+
+// makePatternHandler returns a function that replaces $0, $1, etc. in the pattern with the corresponding match
+func makePatternHandler(pattern string) func([]string) string {
+	return func(matches []string) string {
+		return varFinder.ReplaceAllStringFunc(pattern, func(s string) string {
+			i, err := strconv.Atoi(s[1:])
+			if i >= len(matches) || err != nil {
+				// Return the original placeholder if the index is out of bounds
+				// or the Atoi fails, though given the varFinder regex it should
+				// not be possible.
+				return s
+			}
+			return matches[i]
+		})
+	}
+}
+
+// newMogrifierMapFromEnv loads mogrifiers from environment variables
+// keys is a list of mogrifier names to load
+func newMogrifierMapFromEnv(keys []string) (mogrifierMap, error) {
+	mogrifiers := mogrifierMap{}
+
+	type config struct {
+		Pattern string            `envconfig:"PATTERN"`
+		Tags    map[string]string `envconfig:"TAGS"`
+		Name    string            `envconfig:"NAME"`
+	}
+
+	for _, mogrifier := range keys {
+		cfg := config{}
+		if err := envconfig.Process(envPrefix+"_"+mogrifier, &cfg); err != nil {
+			return nil, fmt.Errorf("failed to load mogrifier %s: %v", mogrifier, err)
+		}
+
+		if cfg.Pattern == "" {
+			return nil, fmt.Errorf("no PATTERN specified for mogrifier %s", mogrifier)
+		}
+
+		re, err := regexp.Compile(cfg.Pattern)
+		if err != nil {
+			return nil, fmt.Errorf("failed to compile pattern for %s: %s: %v", mogrifier, cfg.Pattern, err)
+		}
+
+		if cfg.Name == "" {
+			return nil, fmt.Errorf("no NAME specified for mogrifier %s", mogrifier)
+		}
+
+		nameHandler := makePatternHandler(cfg.Name)
+		tagHandlers := make(map[string]func([]string) string, len(cfg.Tags))
+		for key, value := range cfg.Tags {
+			if key == "" {
+				return nil, fmt.Errorf("no key specified for tag %s for mogrifier %s", key, mogrifier)
+			}
+			tagHandlers[key] = makePatternHandler(value)
+			if value == "" {
+				return nil, fmt.Errorf("no value specified for tag %s for mogrifier %s", key, mogrifier)
+			}
+		}
+
+		mogrifiers[re] = func(matches []string) (string, []string) {
+			name := nameHandler(matches)
+			tags := make([]string, 0, len(tagHandlers))
+			for tagKey, handler := range tagHandlers {
+				tagValue := handler(matches)
+				tags = append(tags, tagKey+":"+tagValue)
+			}
+			return name, tags
+		}
+
+	}
+	return mogrifiers, nil
+}
+
+// mogrify applies the first mogrifier in the map that matches the name
+func (m mogrifierMap) mogrify(name string) (string, []string) {
+	if m == nil {
+		return name, nil
+	}
+	for matcher, mogrifier := range m {
+		matches := matcher.FindStringSubmatch(name)
+		if len(matches) == 0 {
+			continue
+		}
+
+		mogrifiedName, tags := mogrifier(matches)
+		return mogrifiedName, tags
+	}
+
+	// no mogrification
+	return name, nil
+}