From 016843786aaccc8cf39b70ce5bc4d21b7ca446cb Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Thu, 4 Apr 2024 13:43:49 -0400 Subject: [PATCH] Add exemplar support to the prometheus exporter (#5111) --- CHANGELOG.md | 1 + exporters/prometheus/exporter.go | 41 ++++++++- exporters/prometheus/exporter_test.go | 117 ++++++++++++++++++++++++++ exporters/prometheus/go.mod | 2 +- 4 files changed, 159 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc6aaa7f62a..67539c84507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Add `otel.scope.name` and `otel.scope.version` tags to spans exported by `go.opentelemetry.io/otel/exporters/zipkin`. (#5108) - Add support for `AddLink` to `go.opentelemetry.io/otel/bridge/opencensus`. (#5116) - Add `String` method to `Value` and `KeyValue` in `go.opentelemetry.io/otel/log`. (#5117) +- Add Exemplar support to `go.opentelemetry.io/otel/exporters/prometheus`. (#5111) ### Changed diff --git a/exporters/prometheus/exporter.go b/exporters/prometheus/exporter.go index ea09390c5ba..1a8e28542e7 100644 --- a/exporters/prometheus/exporter.go +++ b/exporters/prometheus/exporter.go @@ -5,6 +5,7 @@ package prometheus // import "go.opentelemetry.io/otel/exporters/prometheus" import ( "context" + "encoding/hex" "errors" "fmt" "slices" @@ -32,6 +33,9 @@ const ( scopeInfoMetricName = "otel_scope_info" scopeInfoDescription = "Instrumentation Scope metadata" + + traceIDExemplarKey = "trace_id" + spanIDExemplarKey = "span_id" ) var ( @@ -238,7 +242,6 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { } func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogram metricdata.Histogram[N], m metricdata.Metrics, ks, vs [2]string, name string, resourceKV keyVals) { - // TODO(https://github.com/open-telemetry/opentelemetry-go/issues/3163): support exemplars for _, dp := range histogram.DataPoints { keys, values := getAttrs(dp.Attributes, ks, vs, resourceKV) @@ -255,6 +258,7 @@ func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogra otel.Handle(err) continue } + m = addExemplars(m, dp.Exemplars) ch <- m } } @@ -274,6 +278,7 @@ func addSumMetric[N int64 | float64](ch chan<- prometheus.Metric, sum metricdata otel.Handle(err) continue } + m = addExemplars(m, dp.Exemplars) ch <- m } } @@ -549,3 +554,37 @@ func (c *collector) validateMetrics(name, description string, metricType *dto.Me return false, "" } + +func addExemplars[N int64 | float64](m prometheus.Metric, exemplars []metricdata.Exemplar[N]) prometheus.Metric { + if len(exemplars) == 0 { + return m + } + promExemplars := make([]prometheus.Exemplar, len(exemplars)) + for i, exemplar := range exemplars { + labels := attributesToLabels(exemplar.FilteredAttributes) + // Overwrite any existing trace ID or span ID attributes + labels[traceIDExemplarKey] = hex.EncodeToString(exemplar.TraceID[:]) + labels[spanIDExemplarKey] = hex.EncodeToString(exemplar.SpanID[:]) + promExemplars[i] = prometheus.Exemplar{ + Value: float64(exemplar.Value), + Timestamp: exemplar.Time, + Labels: labels, + } + } + metricWithExemplar, err := prometheus.NewMetricWithExemplars(m, promExemplars...) + if err != nil { + // If there are errors creating the metric with exemplars, just warn + // and return the metric without exemplars. + otel.Handle(err) + return m + } + return metricWithExemplar +} + +func attributesToLabels(attrs []attribute.KeyValue) prometheus.Labels { + labels := make(map[string]string) + for _, attr := range attrs { + labels[string(attr.Key)] = attr.Value.Emit() + } + return labels +} diff --git a/exporters/prometheus/exporter_test.go b/exporters/prometheus/exporter_test.go index ee052604eb1..67efaf9cc9c 100644 --- a/exporters/prometheus/exporter_test.go +++ b/exporters/prometheus/exporter_test.go @@ -13,6 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +23,7 @@ import ( "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "go.opentelemetry.io/otel/trace" ) func TestPrometheusExporter(t *testing.T) { @@ -898,3 +900,118 @@ func TestShutdownExporter(t *testing.T) { // ensure we aren't unnecessarily logging errors from the shutdown MeterProvider require.NoError(t, handledError) } + +func TestExemplars(t *testing.T) { + attrsOpt := otelmetric.WithAttributes( + attribute.Key("A").String("B"), + attribute.Key("C").String("D"), + attribute.Key("E").Bool(true), + attribute.Key("F").Int(42), + ) + for _, tc := range []struct { + name string + recordMetrics func(ctx context.Context, meter otelmetric.Meter) + expectedExemplarValue float64 + }{ + { + name: "counter", + recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { + counter, err := meter.Float64Counter("foo") + require.NoError(t, err) + counter.Add(ctx, 9, attrsOpt) + }, + expectedExemplarValue: 9, + }, + { + name: "histogram", + recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { + hist, err := meter.Int64Histogram("foo") + require.NoError(t, err) + hist.Record(ctx, 9, attrsOpt) + }, + expectedExemplarValue: 9, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("OTEL_GO_X_EXEMPLAR", "true") + // initialize registry exporter + ctx := context.Background() + registry := prometheus.NewRegistry() + exporter, err := New(WithRegisterer(registry), WithoutTargetInfo(), WithoutScopeInfo()) + require.NoError(t, err) + + // initialize resource + res, err := resource.New(ctx, + resource.WithAttributes(semconv.ServiceName("prometheus_test")), + resource.WithAttributes(semconv.TelemetrySDKVersion("latest")), + ) + require.NoError(t, err) + res, err = resource.Merge(resource.Default(), res) + require.NoError(t, err) + + // initialize provider and meter + provider := metric.NewMeterProvider( + metric.WithReader(exporter), + metric.WithResource(res), + metric.WithView(metric.NewView( + metric.Instrument{Name: "*"}, + metric.Stream{ + // filter out all attributes so they are added as filtered + // attributes to the exemplar + AttributeFilter: attribute.NewAllowKeysFilter(), + }, + )), + ) + meter := provider.Meter("meter", otelmetric.WithInstrumentationVersion("v0.1.0")) + + // Add a sampled span context so that measurements get exemplars added + sc := trace.NewSpanContext(trace.SpanContextConfig{ + SpanID: trace.SpanID{0o1}, + TraceID: trace.TraceID{0o1}, + TraceFlags: trace.FlagsSampled, + }) + ctx = trace.ContextWithSpanContext(ctx, sc) + // Record a single observation with the exemplar + tc.recordMetrics(ctx, meter) + + // Verify that the exemplar is present in the proto version of the + // prometheus metrics. + got, done, err := prometheus.ToTransactionalGatherer(registry).Gather() + defer done() + require.NoError(t, err) + + require.Len(t, got, 1) + family := got[0] + require.Len(t, family.GetMetric(), 1) + metric := family.GetMetric()[0] + var exemplar *dto.Exemplar + switch family.GetType() { + case dto.MetricType_COUNTER: + exemplar = metric.GetCounter().GetExemplar() + case dto.MetricType_HISTOGRAM: + for _, b := range metric.GetHistogram().GetBucket() { + if b.GetExemplar() != nil { + exemplar = b.GetExemplar() + continue + } + } + } + require.NotNil(t, exemplar) + require.Equal(t, exemplar.GetValue(), tc.expectedExemplarValue) + expectedLabels := map[string]string{ + traceIDExemplarKey: "01000000000000000000000000000000", + spanIDExemplarKey: "0100000000000000", + "A": "B", + "C": "D", + "E": "true", + "F": "42", + } + require.Equal(t, len(expectedLabels), len(exemplar.GetLabel())) + for _, label := range exemplar.GetLabel() { + val, ok := expectedLabels[label.GetName()] + require.True(t, ok) + require.Equal(t, label.GetValue(), val) + } + }) + } +} diff --git a/exporters/prometheus/go.mod b/exporters/prometheus/go.mod index 51c44b2ade1..7c2f7203483 100644 --- a/exporters/prometheus/go.mod +++ b/exporters/prometheus/go.mod @@ -10,6 +10,7 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 go.opentelemetry.io/otel/sdk v1.24.0 go.opentelemetry.io/otel/sdk/metric v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 google.golang.org/protobuf v1.33.0 ) @@ -23,7 +24,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/sys v0.18.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect )