[prometheusremotewriteexporter] Support sending trace id and span id for exemplars

CHANGELOG.md

@@ -21,6 +21,7 @@
 - `googlecloudpubsubreceiver` Added implementation of Google Cloud Pubsub receiver. (#8391)
 - `googlecloudpubsubexporter` Added implementation of Google Cloud Pubsub exporter. (#8391)
 - `coralogixexporter` Allow exporter timeout to be configured (#7957)
+- `prometheusremotewriteexporter` support adding trace id and span id attached to exemplars (#8380)
 
 ### 🛑 Breaking changes 🛑
 

pkg/translator/prometheusremotewrite/helper.go

@@ -46,6 +46,10 @@ const (
 	// according to the prometheus specification
 	// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
 	maxExemplarRunes = 128
+	// Trace and Span id keys are defined as part of the spec:
+	// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification%2Fmetrics%2Fdatamodel.md#exemplars-2
+	traceIDKey = "trace_id"
+	spanIDKey  = "span_id"
 )
 
 type bucketBoundsData struct {
@@ -357,13 +361,32 @@ func getPromExemplars(pt pdata.HistogramDataPoint) []prompb.Exemplar {
 
 	for i := 0; i < pt.Exemplars().Len(); i++ {
 		exemplar := pt.Exemplars().At(i)
+		exemplarRunes := 0
 
 		promExemplar := &prompb.Exemplar{
 			Value:     exemplar.DoubleVal(),
 			Timestamp: timestamp.FromTime(exemplar.Timestamp().AsTime()),
 		}
+		if !exemplar.TraceID().IsEmpty() {
+			val := exemplar.TraceID().HexString()
+			exemplarRunes += utf8.RuneCountInString(traceIDKey) + utf8.RuneCountInString(val)
+			promLabel := prompb.Label{
+				Name:  traceIDKey,
+				Value: val,
+			}
+			promExemplar.Labels = append(promExemplar.Labels, promLabel)
+		}
+		if !exemplar.SpanID().IsEmpty() {
+			val := exemplar.SpanID().HexString()
+			exemplarRunes += utf8.RuneCountInString(spanIDKey) + utf8.RuneCountInString(val)
+			promLabel := prompb.Label{
+				Name:  spanIDKey,
+				Value: val,
+			}
+			promExemplar.Labels = append(promExemplar.Labels, promLabel)
+		}
+		var labelsFromAttributes []prompb.Label
 
-		exemplarRunes := 0
 		exemplar.FilteredAttributes().Range(func(key string, value pdata.AttributeValue) bool {
 			val := value.AsString()
 			exemplarRunes += utf8.RuneCountInString(key) + utf8.RuneCountInString(val)
@@ -372,12 +395,14 @@ func getPromExemplars(pt pdata.HistogramDataPoint) []prompb.Exemplar {
 				Value: val,
 			}
 
-			promExemplar.Labels = append(promExemplar.Labels, promLabel)
+			labelsFromAttributes = append(labelsFromAttributes, promLabel)
 
 			return true
 		})
-		if exemplarRunes > maxExemplarRunes {
-			promExemplar.Labels = nil
+		if exemplarRunes <= maxExemplarRunes {
+			// only append filtered attributes if it does not cause exemplar
+			// labels to exceed the max number of runes
+			promExemplar.Labels = append(promExemplar.Labels, labelsFromAttributes...)
 		}
 
 		promExemplars = append(promExemplars, *promExemplar)

pkg/translator/prometheusremotewrite/helper_test.go

@@ -424,18 +424,29 @@ func Test_getPromExemplars(t *testing.T) {
 	}{
 		{
 			"with_exemplars",
-			getHistogramDataPointWithExemplars(tnow, floatVal1, traceIDKey, traceIDValue1),
+			getHistogramDataPointWithExemplars(t, tnow, floatVal1, traceIDValue1, spanIDValue1, label11, value11),
 			[]prompb.Exemplar{
 				{
 					Value:     floatVal1,
 					Timestamp: timestamp.FromTime(tnow),
-					Labels:    []prompb.Label{getLabel(traceIDKey, traceIDValue1)},
+					Labels:    []prompb.Label{getLabel(traceIDKey, traceIDValue1), getLabel(spanIDKey, spanIDValue1), getLabel(label11, value11)},
+				},
+			},
+		},
+		{
+			"with_exemplars_without_trace_or_span",
+			getHistogramDataPointWithExemplars(t, tnow, floatVal1, "", "", label11, value11),
+			[]prompb.Exemplar{
+				{
+					Value:     floatVal1,
+					Timestamp: timestamp.FromTime(tnow),
+					Labels:    []prompb.Label{getLabel(label11, value11)},
 				},
 			},
 		},
 		{
 			"too_many_runes_drops_labels",
-			getHistogramDataPointWithExemplars(tnow, floatVal1, keyWith129Runes, ""),
+			getHistogramDataPointWithExemplars(t, tnow, floatVal1, "", "", keyWith129Runes, ""),
 			[]prompb.Exemplar{
 				{
 					Value:     floatVal1,
@@ -445,7 +456,7 @@ func Test_getPromExemplars(t *testing.T) {
 		},
 		{
 			"runes_at_limit_bytes_over_keeps_labels",
-			getHistogramDataPointWithExemplars(tnow, floatVal1, keyWith128Runes, ""),
+			getHistogramDataPointWithExemplars(t, tnow, floatVal1, "", "", keyWith128Runes, ""),
 			[]prompb.Exemplar{
 				{
 					Value:     floatVal1,
@@ -454,6 +465,17 @@ func Test_getPromExemplars(t *testing.T) {
 				},
 			},
 		},
+		{
+			"too_many_runes_with_exemplar_drops_attrs_keeps_exemplar",
+			getHistogramDataPointWithExemplars(t, tnow, floatVal1, traceIDValue1, spanIDValue1, keyWith64Runes, ""),
+			[]prompb.Exemplar{
+				{
+					Value:     floatVal1,
+					Timestamp: timestamp.FromTime(tnow),
+					Labels:    []prompb.Label{getLabel(traceIDKey, traceIDValue1), getLabel(spanIDKey, spanIDValue1)},
+				},
+			},
+		},
 		{
 			"without_exemplar",
 			getHistogramDataPoint(),

pkg/translator/prometheusremotewrite/testutils_test.go

@@ -15,11 +15,14 @@
 package prometheusremotewrite
 
 import (
+	"encoding/hex"
 	"math"
 	"strings"
+	"testing"
 	"time"
 
 	"github.com/prometheus/prometheus/prompb"
+	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/model/pdata"
 )
 
@@ -45,14 +48,16 @@ var (
 	value41            = "test_value41"
 	dirty1             = "%"
 	dirty2             = "?"
-	traceIDValue1      = "traceID-value1"
-	traceIDKey         = "trace_id"
+	traceIDValue1      = "4303853f086f4f8c86cf198b6551df84"
+	spanIDValue1       = "e5513c32795c41b9"
 	colliding1         = "test.colliding"
 	colliding2         = "test/colliding"
 	collidingSanitized = "test_colliding"
 	keyWith129Runes    = "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii"
 	// because of the special characters, this has 132 bytes and 128 runes
 	keyWith128Runes = "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii世界"
+	// 64 + trace id + span id = 129 characters
+	keyWith64Runes = "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii"
 
 	intVal1   int64 = 1
 	intVal2   int64 = 2
@@ -221,14 +226,29 @@ func getTimeSeriesWithSamplesAndExemplars(labels []prompb.Label, samples []promp
 	}
 }
 
-func getHistogramDataPointWithExemplars(time time.Time, value float64, attributeKey string, attributeValue string) *pdata.HistogramDataPoint {
+func getHistogramDataPointWithExemplars(t *testing.T, time time.Time, value float64, traceID string, spanID string, attributeKey string, attributeValue string) *pdata.HistogramDataPoint {
 	h := pdata.NewHistogramDataPoint()
 
 	e := h.Exemplars().AppendEmpty()
 	e.SetDoubleVal(value)
 	e.SetTimestamp(pdata.NewTimestampFromTime(time))
 	e.FilteredAttributes().Insert(attributeKey, pdata.NewAttributeValueString(attributeValue))
 
+	if traceID != "" {
+		var traceIDBytes [16]byte
+		traceIDBytesSlice, err := hex.DecodeString(traceID)
+		require.NoErrorf(t, err, "error decoding trace id: %v", err)
+		copy(traceIDBytes[:], traceIDBytesSlice)
+		e.SetTraceID(pdata.NewTraceID(traceIDBytes))
+	}
+	if spanID != "" {
+		var spanIDBytes [8]byte
+		spanIDBytesSlice, err := hex.DecodeString(spanID)
+		require.NoErrorf(t, err, "error decoding span id: %v", err)
+		copy(spanIDBytes[:], spanIDBytesSlice)
+		e.SetSpanID(pdata.NewSpanID(spanIDBytes))
+	}
+
 	return &h
 }