[prometheusremotewriteexporter] Allow to disable sanitize metric labels

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 - `googlecloudexporter`: [Alpha] Translate metrics directly from OTLP to gcm using the `exporter.googlecloud.OTLPDirect` feature-gate (#7177)
 - `simpleprometheusreceiver`: Add support for static labels (#7908)
+- `prometheusremotewriteexporter`: Allow to disable sanitize metric labels (#8270)
 
 ### 🛑 Breaking changes 🛑
 

exporter/prometheusremotewriteexporter/exporter.go

@@ -44,6 +44,7 @@ const maxBatchByteSize = 3000000
 // prwExporter converts OTLP metrics to Prometheus remote write TimeSeries and sends them to a remote endpoint.
 type prwExporter struct {
 	namespace       string
+	sanitizeLabel   bool
 	externalLabels  map[string]string
 	endpointURL     *url.URL
 	client          *http.Client
@@ -71,6 +72,7 @@ func newPRWExporter(cfg *Config, set component.ExporterCreateSettings) (*prwExpo
 
 	return &prwExporter{
 		namespace:       cfg.Namespace,
+		sanitizeLabel:   cfg.sanitizeLabel,
 		externalLabels:  sanitizedLabels,
 		endpointURL:     endpointURL,
 		wg:              new(sync.WaitGroup),
@@ -107,7 +109,7 @@ func (prwe *prwExporter) PushMetrics(ctx context.Context, md pdata.Metrics) erro
 	case <-prwe.closeChan:
 		return errors.New("shutdown has been called")
 	default:
-		tsMap, err := prometheusremotewrite.FromMetrics(md, prometheusremotewrite.Settings{Namespace: prwe.namespace, ExternalLabels: prwe.externalLabels})
+		tsMap, err := prometheusremotewrite.FromMetrics(md, prometheusremotewrite.Settings{Namespace: prwe.namespace, ExternalLabels: prwe.externalLabels, SanitizeLabel: prwe.sanitizeLabel})
 		if err != nil {
 			err = consumererror.NewPermanent(err)
 		}

pkg/translator/prometheusremotewrite/helper.go

@@ -145,7 +145,7 @@ func timeSeriesSignature(metric pdata.Metric, labels *[]prompb.Label) string {
 // createAttributes creates a slice of Cortex Label with OTLP attributes and pairs of string values.
 // Unpaired string value is ignored. String pairs overwrites OTLP labels if collision happens, and the overwrite is
 // logged. Resultant label names are sanitized.
-func createAttributes(resource pdata.Resource, attributes pdata.AttributeMap, externalLabels map[string]string, extras ...string) []prompb.Label {
+func createAttributes(resource pdata.Resource, attributes pdata.AttributeMap, externalLabels map[string]string, sanitizeLabel bool, extras ...string) []prompb.Label {
 	// map ensures no duplicate label name
 	l := map[string]prompb.Label{}
 
@@ -159,19 +159,33 @@ func createAttributes(resource pdata.Resource, attributes pdata.AttributeMap, ex
 
 	resource.Attributes().Range(func(key string, value pdata.AttributeValue) bool {
 		if isUsefulResourceAttribute(key) {
-			l[key] = prompb.Label{
-				Name:  sanitize(key),
-				Value: value.AsString(),
+			if sanitizeLabel {
+				l[key] = prompb.Label{
+					Name:  sanitize(key),
+					Value: value.AsString(),
+				}
+			} else {
+				l[key] = prompb.Label{
+					Name:  sanitizeLabels(key),
+					Value: value.AsString(),
+				}
 			}
 		}
 
 		return true
 	})
 
 	attributes.Range(func(key string, value pdata.AttributeValue) bool {
-		l[key] = prompb.Label{
-			Name:  sanitize(key),
-			Value: value.AsString(),
+		if sanitizeLabel {
+			l[key] = prompb.Label{
+				Name:  sanitize(key),
+				Value: value.AsString(),
+			}
+		} else {
+			l[key] = prompb.Label{
+				Name:  sanitizeLabels(key),
+				Value: value.AsString(),
+			}
 		}
 
 		return true
@@ -249,7 +263,7 @@ func validateMetrics(metric pdata.Metric) bool {
 func addSingleNumberDataPoint(pt pdata.NumberDataPoint, resource pdata.Resource, metric pdata.Metric, settings Settings, tsMap map[string]*prompb.TimeSeries) {
 	// create parameters for addSample
 	name := getPromMetricName(metric, settings.Namespace)
-	labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, name)
+	labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, name)
 	sample := &prompb.Sample{
 		// convert ns to ms
 		Timestamp: convertTimeStamp(pt.Timestamp()),
@@ -281,7 +295,7 @@ func addSingleHistogramDataPoint(pt pdata.HistogramDataPoint, resource pdata.Res
 		sum.Value = math.Float64frombits(value.StaleNaN)
 	}
 
-	sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr)
+	sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName+sumStr)
 	addSample(tsMap, sum, sumlabels, metric)
 
 	// treat count as a sample in an individual TimeSeries
@@ -293,7 +307,7 @@ func addSingleHistogramDataPoint(pt pdata.HistogramDataPoint, resource pdata.Res
 		count.Value = math.Float64frombits(value.StaleNaN)
 	}
 
-	countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr)
+	countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName+countStr)
 	addSample(tsMap, count, countlabels, metric)
 
 	// cumulative count for conversion to cumulative histogram
@@ -317,7 +331,7 @@ func addSingleHistogramDataPoint(pt pdata.HistogramDataPoint, resource pdata.Res
 			bucket.Value = math.Float64frombits(value.StaleNaN)
 		}
 		boundStr := strconv.FormatFloat(bound, 'f', -1, 64)
-		labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, boundStr)
+		labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName+bucketStr, leStr, boundStr)
 		sig := addSample(tsMap, bucket, labels, metric)
 
 		bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: bound})
@@ -332,7 +346,7 @@ func addSingleHistogramDataPoint(pt pdata.HistogramDataPoint, resource pdata.Res
 		cumulativeCount += pt.BucketCounts()[len(pt.BucketCounts())-1]
 		infBucket.Value = float64(cumulativeCount)
 	}
-	infLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, pInfStr)
+	infLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName+bucketStr, leStr, pInfStr)
 	sig := addSample(tsMap, infBucket, infLabels, metric)
 
 	bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: math.Inf(1)})
@@ -381,7 +395,7 @@ func addSingleSummaryDataPoint(pt pdata.SummaryDataPoint, resource pdata.Resourc
 	if pt.Flags().HasFlag(pdata.MetricDataPointFlagNoRecordedValue) {
 		sum.Value = math.Float64frombits(value.StaleNaN)
 	}
-	sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr)
+	sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName+sumStr)
 	addSample(tsMap, sum, sumlabels, metric)
 
 	// treat count as a sample in an individual TimeSeries
@@ -392,7 +406,7 @@ func addSingleSummaryDataPoint(pt pdata.SummaryDataPoint, resource pdata.Resourc
 	if pt.Flags().HasFlag(pdata.MetricDataPointFlagNoRecordedValue) {
 		count.Value = math.Float64frombits(value.StaleNaN)
 	}
-	countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr)
+	countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName+countStr)
 	addSample(tsMap, count, countlabels, metric)
 
 	// process each percentile/quantile
@@ -406,7 +420,7 @@ func addSingleSummaryDataPoint(pt pdata.SummaryDataPoint, resource pdata.Resourc
 			quantile.Value = math.Float64frombits(value.StaleNaN)
 		}
 		percentileStr := strconv.FormatFloat(qt.Quantile(), 'f', -1, 64)
-		qtlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName, quantileStr, percentileStr)
+		qtlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, settings.SanitizeLabel, nameStr, baseName, quantileStr, percentileStr)
 		addSample(tsMap, quantile, qtlabels, metric)
 	}
 }
@@ -431,6 +445,21 @@ func sanitize(s string) string {
 	return s
 }
 
+func sanitizeLabels(s string) string {
+	if len(s) == 0 {
+		return s
+	}
+
+	// Note: No length limit for label keys because Prometheus doesn't
+	// define a length limit, thus we should NOT be truncating label keys.
+	// labels that start with _ are not sanitized
+	s = strings.Map(sanitizeRune, s)
+	if unicode.IsDigit(rune(s[0])) {
+		s = keyStr + "_" + s
+	}
+	return s
+}
+
 // copied from prometheus-go-metric-exporter
 // sanitizeRune converts anything that is not a letter or digit to an underscore
 func sanitizeRune(r rune) rune {

pkg/translator/prometheusremotewrite/helper_test.go

@@ -184,6 +184,7 @@ func Test_createLabelSet(t *testing.T) {
 		resource       pdata.Resource
 		orig           pdata.AttributeMap
 		externalLabels map[string]string
+		sanitizeLabel  bool
 		extras         []string
 		want           []prompb.Label
 	}{
@@ -192,6 +193,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1,
 			map[string]string{},
+			true,
 			[]string{label31, value31, label32, value32},
 			getPromLabels(label11, value11, label12, value12, label31, value31, label32, value32),
 		},
@@ -203,6 +205,7 @@ func Test_createLabelSet(t *testing.T) {
 			}),
 			lbs1,
 			map[string]string{},
+			true,
 			[]string{label31, value31, label32, value32},
 			getPromLabels(label11, value11, label12, value12, label31, value31, label32, value32, "job", "prometheus", "instance", "127.0.0.1:8080"),
 		},
@@ -214,6 +217,7 @@ func Test_createLabelSet(t *testing.T) {
 			}),
 			lbs1,
 			map[string]string{},
+			true,
 			[]string{label31, value31, label32, value32},
 			getPromLabels(label11, value11, label12, value12, label31, value31, label32, value32, "job", "12345", "instance", "true"),
 		},
@@ -222,6 +226,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1,
 			map[string]string{},
+			true,
 			[]string{label11, value31},
 			getPromLabels(label11, value31, label12, value12),
 		},
@@ -230,6 +235,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1Dirty,
 			map[string]string{},
+			true,
 			[]string{label31 + dirty1, value31, label32, value32},
 			getPromLabels(label11+"_", value11, "key_"+label12, value12, label31+"_", value31, label32, value32),
 		},
@@ -238,6 +244,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			pdata.NewAttributeMap(),
 			nil,
+			true,
 			[]string{label31, value31, label32, value32},
 			getPromLabels(label31, value31, label32, value32),
 		},
@@ -246,6 +253,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1,
 			map[string]string{},
+			true,
 			[]string{"", ""},
 			getPromLabels(label11, value11, label12, value12, "", ""),
 		},
@@ -254,6 +262,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1,
 			map[string]string{},
+			true,
 			[]string{label31, value31, label32},
 			getPromLabels(label11, value11, label12, value12, label31, value31),
 		},
@@ -262,6 +271,7 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1,
 			exlbs1,
+			true,
 			[]string{label31, value31, label32, value32},
 			getPromLabels(label11, value11, label12, value12, label41, value41, label31, value31, label32, value32),
 		},
@@ -270,14 +280,33 @@ func Test_createLabelSet(t *testing.T) {
 			getResource(map[string]pdata.AttributeValue{}),
 			lbs1,
 			exlbs2,
+			true,
 			[]string{label31, value31, label32, value32},
 			getPromLabels(label11, value11, label12, value12, label31, value31, label32, value32),
 		},
+		{
+			"sanitize_labels_starts_with_underscore",
+			getResource(map[string]pdata.AttributeValue{}),
+			lbs3,
+			exlbs1,
+			true,
+			[]string{label31, value31, label32, value32},
+			getPromLabels(label11, value11, label12, value12, keyStr+label51, value51, label41, value41, label31, value31, label32, value32),
+		},
+		{
+			"donot_sanitize_labels_starts_with_underscore",
+			getResource(map[string]pdata.AttributeValue{}),
+			lbs3,
+			exlbs1,
+			false,
+			[]string{label31, value31, label32, value32},
+			getPromLabels(label11, value11, label12, value12, label51, value51, label41, value41, label31, value31, label32, value32),
+		},
 	}
 	// run tests
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assert.ElementsMatch(t, tt.want, createAttributes(tt.resource, tt.orig, tt.externalLabels, tt.extras...))
+			assert.ElementsMatch(t, tt.want, createAttributes(tt.resource, tt.orig, tt.externalLabels, tt.sanitizeLabel, tt.extras...))
 		})
 	}
 }

pkg/translator/prometheusremotewrite/metrics_to_prw.go

@@ -26,7 +26,7 @@ import (
 
 // Deprecated: [0.45.0] use `prometheusremotewrite.FromMetrics`. It does not wrap the error as `NewPermanent`.
 func MetricsToPRW(namespace string, externalLabels map[string]string, md pdata.Metrics) (map[string]*prompb.TimeSeries, int, error) {
-	tsMap, err := FromMetrics(md, Settings{Namespace: namespace, ExternalLabels: externalLabels})
+	tsMap, err := FromMetrics(md, Settings{Namespace: namespace, ExternalLabels: externalLabels, SanitizeLabel: true})
 	if err != nil {
 		err = consumererror.NewPermanent(err)
 	}
@@ -36,6 +36,7 @@ func MetricsToPRW(namespace string, externalLabels map[string]string, md pdata.M
 type Settings struct {
 	Namespace      string
 	ExternalLabels map[string]string
+	SanitizeLabel  bool
 }
 
 // FromMetrics converts pdata.Metrics to prometheus remote write format.

pkg/translator/prometheusremotewrite/testutils_test.go

@@ -43,6 +43,8 @@ var (
 	value32       = "test_value32"
 	label41       = "__test_label41__"
 	value41       = "test_value41"
+	label51       = "_test_label51"
+	value51       = "test_value51"
 	dirty1        = "%"
 	dirty2        = "?"
 	traceIDValue1 = "traceID-value1"
@@ -55,6 +57,7 @@ var (
 
 	lbs1      = getAttributes(label11, value11, label12, value12)
 	lbs2      = getAttributes(label21, value21, label22, value22)
+	lbs3      = getAttributes(label11, value11, label12, value12, label51, value51)
 	lbs1Dirty = getAttributes(label11+dirty1, value11, dirty2+label12, value12)
 
 	exlbs1 = map[string]string{label41: value41}