make the traceID label name configurable

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## main / unreleased
 
+* [ENHANCEMENT] Make the trace ID label name configurable [#3074](https://github.com/grafana/tempo/pull/3074)
+
 ## v2.3.0-rc.0 / 2023-10-20
 
 * [CHANGE] Update Go to 1.21 [#2486](https://github.com/grafana/tempo/pull/2829) (@zalegrala)

docs/sources/tempo/configuration/_index.md

@@ -1354,6 +1354,10 @@ overrides:
       # This setting is useful if you wish to test how many active series a tenant will generate, without
       # actually writing these metrics.
       [disable_collection: <bool> | default = false]
+
+      # Per-user configuration of the trace-id label name. This value will be used as name for the label to store the
+      # trace ID of exemplars in generated metrics. If not set, the default value "trace_id" will be used.  
+      [trace_id_label_name: <string> | default = "trace_id"]  
 
       # This option only allows spans with end time that occur within the configured duration to be
       # considered in metrics generation.

modules/generator/overrides_test.go

@@ -52,6 +52,10 @@ func (m *mockOverrides) MetricsGeneratorDisableCollection(string) bool {
 	return false
 }
 
+func (m *mockOverrides) MetricsGenerationTraceIDLabelName(userID string) string {
+	return ""
+}
+
 func (m *mockOverrides) MetricsGeneratorProcessorServiceGraphsHistogramBuckets(string) []float64 {
 	return m.serviceGraphsHistogramBuckets
 }

modules/generator/registry/counter.go

@@ -115,7 +115,7 @@ func (c *counter) name() string {
 	return c.metricName
 }
 
-func (c *counter) collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string) (activeSeries int, err error) {
+func (c *counter) collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string, _ string) (activeSeries int, err error) {
 	c.seriesMtx.RLock()
 	defer c.seriesMtx.RUnlock()
 

modules/generator/registry/counter_test.go

@@ -210,7 +210,7 @@ func Test_counter_concurrencyDataRace(t *testing.T) {
 	})
 
 	go accessor(func() {
-		_, err := c.collectMetrics(&noopAppender{}, 0, nil)
+		_, err := c.collectMetrics(&noopAppender{}, 0, nil, "")
 		assert.NoError(t, err)
 	})
 
@@ -263,7 +263,7 @@ func Test_counter_concurrencyCorrectness(t *testing.T) {
 func collectMetricAndAssert(t *testing.T, m metric, collectionTimeMs int64, externalLabels map[string]string, expectedActiveSeries int, expectedSamples []sample, expectedExemplars []exemplarSample) {
 	appender := &capturingAppender{}
 
-	activeSeries, err := m.collectMetrics(appender, collectionTimeMs, externalLabels)
+	activeSeries, err := m.collectMetrics(appender, collectionTimeMs, externalLabels, "traceID")
 	assert.NoError(t, err)
 	assert.Equal(t, expectedActiveSeries, activeSeries)
 

modules/generator/registry/gauge.go

@@ -123,7 +123,7 @@ func (g *gauge) name() string {
 	return g.metricName
 }
 
-func (g *gauge) collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string) (activeSeries int, err error) {
+func (g *gauge) collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string, _ string) (activeSeries int, err error) {
 	g.seriesMtx.RLock()
 	defer g.seriesMtx.RUnlock()
 

modules/generator/registry/gauge_test.go

@@ -227,7 +227,7 @@ func Test_gauge_concurrencyDataRace(t *testing.T) {
 	})
 
 	go accessor(func() {
-		_, err := c.collectMetrics(&noopAppender{}, 0, nil)
+		_, err := c.collectMetrics(&noopAppender{}, 0, nil, "")
 		assert.NoError(t, err)
 	})
 

modules/generator/registry/histogram.go

@@ -149,7 +149,7 @@ func (h *histogram) name() string {
 	return h.metricName
 }
 
-func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string) (activeSeries int, err error) {
+func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string, traceIDLabelName string) (activeSeries int, err error) {
 	h.seriesMtx.RLock()
 	defer h.seriesMtx.RUnlock()
 
@@ -201,7 +201,7 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte
 			if ex != "" {
 				_, err = appender.AppendExemplar(ref, lb.Labels(), exemplar.Exemplar{
 					Labels: []labels.Label{{
-						Name:  "traceID",
+						Name:  traceIDLabelName,
 						Value: ex,
 					}},
 					Value: s.exemplarValues[i].Load(),

modules/generator/registry/histogram_test.go

@@ -295,7 +295,7 @@ func Test_histogram_concurrencyDataRace(t *testing.T) {
 	})
 
 	go accessor(func() {
-		_, err := h.collectMetrics(&noopAppender{}, 0, nil)
+		_, err := h.collectMetrics(&noopAppender{}, 0, nil, "")
 		assert.NoError(t, err)
 	})
 

modules/generator/registry/overrides.go

@@ -10,6 +10,7 @@ type Overrides interface {
 	MetricsGeneratorMaxActiveSeries(userID string) uint32
 	MetricsGeneratorCollectionInterval(userID string) time.Duration
 	MetricsGeneratorDisableCollection(userID string) bool
+	MetricsGenerationTraceIDLabelName(userID string) string
 }
 
 var _ Overrides = (overrides.Interface)(nil)

modules/generator/registry/registry.go

@@ -83,7 +83,7 @@ type ManagedRegistry struct {
 // metric is the interface for a metric that is managed by ManagedRegistry.
 type metric interface {
 	name() string
-	collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string) (activeSeries int, err error)
+	collectMetrics(appender storage.Appender, timeMs int64, externalLabels map[string]string, traceIDLabelName string) (activeSeries int, err error)
 	removeStaleSeries(staleTimeMs int64)
 }
 
@@ -192,6 +192,11 @@ func (r *ManagedRegistry) collectMetrics(ctx context.Context) {
 		return
 	}
 
+	traceIDLabelName := r.overrides.MetricsGenerationTraceIDLabelName(r.tenant)
+	if traceIDLabelName == "" {
+		traceIDLabelName = "traceID"
+	}
+
 	r.metricsMtx.RLock()
 	defer r.metricsMtx.RUnlock()
 
@@ -210,7 +215,7 @@ func (r *ManagedRegistry) collectMetrics(ctx context.Context) {
 	collectionTimeMs := time.Now().UnixMilli()
 
 	for _, m := range r.metrics {
-		active, err := m.collectMetrics(appender, collectionTimeMs, r.externalLabels)
+		active, err := m.collectMetrics(appender, collectionTimeMs, r.externalLabels, traceIDLabelName)
 		if err != nil {
 			return
 		}

modules/generator/registry/registry_test.go

@@ -297,6 +297,10 @@ func (m *mockOverrides) MetricsGeneratorDisableCollection(string) bool {
 	return m.disableCollection
 }
 
+func (m *mockOverrides) MetricsGenerationTraceIDLabelName(string) string {
+	return ""
+}
+
 func mustGetHostname() string {
 	hostname, _ := os.Hostname()
 	return hostname

modules/overrides/config.go

@@ -111,6 +111,7 @@ type MetricsGeneratorOverrides struct {
 	MaxActiveSeries    uint32              `yaml:"max_active_series,omitempty" json:"max_active_series,omitempty"`
 	CollectionInterval time.Duration       `yaml:"collection_interval,omitempty" json:"collection_interval,omitempty"`
 	DisableCollection  bool                `yaml:"disable_collection,omitempty" json:"disable_collection,omitempty"`
+	TraceIDLabelName   string              `yaml:"trace_id_label_name,omitempty" json:"trace_id_label_name,omitempty"`
 
 	Forwarder ForwarderOverrides `yaml:"forwarder,omitempty" json:"forwarder,omitempty"`
 

modules/overrides/config_legacy.go

@@ -27,6 +27,7 @@ func (c *Overrides) toLegacy() LegacyOverrides {
 		MetricsGeneratorMaxActiveSeries:                                  c.MetricsGenerator.MaxActiveSeries,
 		MetricsGeneratorCollectionInterval:                               c.MetricsGenerator.CollectionInterval,
 		MetricsGeneratorDisableCollection:                                c.MetricsGenerator.DisableCollection,
+		MetricsGeneratorTraceIDLabelName:                                 c.MetricsGenerator.TraceIDLabelName,
 		MetricsGeneratorForwarderQueueSize:                               c.MetricsGenerator.Forwarder.QueueSize,
 		MetricsGeneratorForwarderWorkers:                                 c.MetricsGenerator.Forwarder.Workers,
 		MetricsGeneratorProcessorServiceGraphsHistogramBuckets:           c.MetricsGenerator.Processor.ServiceGraphs.HistogramBuckets,
@@ -81,6 +82,7 @@ type LegacyOverrides struct {
 	MetricsGeneratorMaxActiveSeries                                  uint32                           `yaml:"metrics_generator_max_active_series" json:"metrics_generator_max_active_series"`
 	MetricsGeneratorCollectionInterval                               time.Duration                    `yaml:"metrics_generator_collection_interval" json:"metrics_generator_collection_interval"`
 	MetricsGeneratorDisableCollection                                bool                             `yaml:"metrics_generator_disable_collection" json:"metrics_generator_disable_collection"`
+	MetricsGeneratorTraceIDLabelName                                 string                           `yaml:"metrics_generator_trace_id_label_name" json:"metrics_generator_trace_id_label_name"`
 	MetricsGeneratorForwarderQueueSize                               int                              `yaml:"metrics_generator_forwarder_queue_size" json:"metrics_generator_forwarder_queue_size"`
 	MetricsGeneratorForwarderWorkers                                 int                              `yaml:"metrics_generator_forwarder_workers" json:"metrics_generator_forwarder_workers"`
 	MetricsGeneratorProcessorServiceGraphsHistogramBuckets           []float64                        `yaml:"metrics_generator_processor_service_graphs_histogram_buckets" json:"metrics_generator_processor_service_graphs_histogram_buckets"`
@@ -143,6 +145,7 @@ func (l *LegacyOverrides) toNewLimits() Overrides {
 			MaxActiveSeries:    l.MetricsGeneratorMaxActiveSeries,
 			CollectionInterval: l.MetricsGeneratorCollectionInterval,
 			DisableCollection:  l.MetricsGeneratorDisableCollection,
+			TraceIDLabelName:   l.MetricsGeneratorTraceIDLabelName,
 			IngestionSlack:     l.MetricsGeneratorIngestionSlack,
 			Forwarder: ForwarderOverrides{
 				QueueSize: l.MetricsGeneratorForwarderQueueSize,

modules/overrides/interface.go

@@ -37,6 +37,7 @@ type Interface interface {
 	MetricsGeneratorMaxActiveSeries(userID string) uint32
 	MetricsGeneratorCollectionInterval(userID string) time.Duration
 	MetricsGeneratorDisableCollection(userID string) bool
+	MetricsGenerationTraceIDLabelName(userID string) string
 	MetricsGeneratorForwarderQueueSize(userID string) int
 	MetricsGeneratorForwarderWorkers(userID string) int
 	MetricsGeneratorProcessorServiceGraphsHistogramBuckets(userID string) []float64

modules/overrides/runtime_config_overrides.go

@@ -330,6 +330,12 @@ func (o *runtimeConfigOverridesManager) MetricsGeneratorDisableCollection(userID
 	return o.getOverridesForUser(userID).MetricsGenerator.DisableCollection
 }
 
+// MetricsGenerationTraceIDLabelName is the label name used for the trace ID in metrics.
+// "TraceID" is used if no value is provided.
+func (o *runtimeConfigOverridesManager) MetricsGenerationTraceIDLabelName(userID string) string {
+	return o.getOverridesForUser(userID).MetricsGenerator.TraceIDLabelName
+}
+
 // MetricsGeneratorForwarderQueueSize is the size of the buffer of requests to send to the metrics-generator
 // from the distributor for this tenant.
 func (o *runtimeConfigOverridesManager) MetricsGeneratorForwarderQueueSize(userID string) int {

modules/overrides/user_configurable_overrides.go

@@ -215,6 +215,10 @@ func (o *userConfigurableOverridesManager) MetricsGeneratorDisableCollection(use
 	return o.Interface.MetricsGeneratorDisableCollection(userID)
 }
 
+func (o *userConfigurableOverridesManager) MetricsGenerationTraceIDLabelName(userID string) string {
+	return o.Interface.MetricsGenerationTraceIDLabelName(userID)
+}
+
 func (o *userConfigurableOverridesManager) MetricsGeneratorCollectionInterval(userID string) time.Duration {
 	if collectionInterval, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetCollectionInterval(); ok {
 		return collectionInterval

modules/overrides/user_configurable_overrides_test.go

@@ -393,6 +393,7 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) {
 	assert.Equal(t, mgr.MetricsGeneratorMaxActiveSeries(tenantID), baseMgr.MetricsGeneratorMaxActiveSeries(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorCollectionInterval(tenantID), baseMgr.MetricsGeneratorCollectionInterval(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorDisableCollection(tenantID), baseMgr.MetricsGeneratorDisableCollection(tenantID))
+	assert.Equal(t, mgr.MetricsGenerationTraceIDLabelName(tenantID), baseMgr.MetricsGenerationTraceIDLabelName(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorForwarderQueueSize(tenantID), baseMgr.MetricsGeneratorForwarderQueueSize(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorForwarderWorkers(tenantID), baseMgr.MetricsGeneratorForwarderWorkers(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID), baseMgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID))