Add rate limit to metrics and update normalization rules

dart/lib/src/metrics/metric.dart

@@ -4,10 +4,9 @@ import 'package:meta/meta.dart';
 
 import '../../sentry.dart';
 
-final RegExp forbiddenKeyCharsRegex = RegExp('[^a-zA-Z0-9_/.-]+');
-final RegExp forbiddenValueCharsRegex =
-    RegExp('[^\\w\\d\\s_:/@\\.\\{\\}\\[\\]\$-]+');
-final RegExp forbiddenUnitCharsRegex = RegExp('[^a-zA-Z0-9_/.]+');
+final RegExp unitRegex = RegExp('[^\\w]+');
+final RegExp nameRegex = RegExp('[^\\w-.]+');
+final RegExp tagKeyRegex = RegExp('[^\\w-./]+');
 
 /// Base class for metrics.
 /// Each metric is identified by a [key]. Its [type] describes its behaviour.
@@ -69,7 +68,7 @@ abstract class Metric {
   ///  and it's appended at the end of the encoded metric.
   String encodeToStatsd(int bucketKey) {
     final buffer = StringBuffer();
-    buffer.write(_normalizeKey(key));
+    buffer.write(_sanitizeName(key));
     buffer.write("@");
 
     final sanitizeUnitName = _sanitizeUnit(unit.name);
@@ -87,7 +86,7 @@ abstract class Metric {
       buffer.write("|#");
       final serializedTags = tags.entries
           .map((tag) =>
-              '${_normalizeKey(tag.key)}:${_normalizeTagValue(tag.value)}')
+              '${_sanitizeTagKey(tag.key)}:${_sanitizeTagValue(tag.value)}')
           .join(',');
       buffer.write(serializedTags);
     }
@@ -117,16 +116,43 @@ abstract class Metric {
   String getSpanAggregationKey() => '${type.statsdType}:$key@${unit.name}';
 
   /// Remove forbidden characters from the metric key and tag key.
-  String _normalizeKey(String input) =>
-      input.replaceAll(forbiddenKeyCharsRegex, '_');
+  String _sanitizeName(String input) => input.replaceAll(nameRegex, '_');
 
   /// Remove forbidden characters from the tag value.
-  String _normalizeTagValue(String input) =>
-      input.replaceAll(forbiddenValueCharsRegex, '');
+  String _sanitizeTagKey(String input) => input.replaceAll(tagKeyRegex, '');
 
   /// Remove forbidden characters from the metric unit.
-  String _sanitizeUnit(String input) =>
-      input.replaceAll(forbiddenUnitCharsRegex, '_');
+  String _sanitizeUnit(String input) => input.replaceAll(unitRegex, '');
+
+  String _sanitizeTagValue(String input) {
+    // see https://develop.sentry.dev/sdk/metrics/#tag-values-replacement-map
+    // Line feed       -> \n
+    // Carriage return -> \r
+    // Tab             -> \t
+    // Backslash       -> \\
+    // Pipe            -> \\u{7c}
+    // Comma           -> \\u{2c}
+    final buffer = StringBuffer();
+    for (int i = 0; i < input.length; i++) {
+      final ch = input[i];
+      if (ch == '\n') {
+        buffer.write("\\n");
+      } else if (ch == '\r') {
+        buffer.write("\\r");
+      } else if (ch == '\t') {
+        buffer.write("\\t");
+      } else if (ch == '\\') {
+        buffer.write("\\\\");
+      } else if (ch == '|') {
+        buffer.write("\\u{7c}");
+      } else if (ch == ',') {
+        buffer.write("\\u{2c}");
+      } else {
+        buffer.write(ch);
+      }
+    }
+    return buffer.toString();
+  }
 }
 
 /// Metric [MetricType.counter] that tracks a value that can only be incremented.

dart/lib/src/transport/data_category.dart

@@ -7,6 +7,7 @@ enum DataCategory {
   transaction,
   attachment,
   security,
+  metricBucket,
   unknown
 }
 
@@ -27,6 +28,8 @@ extension DataCategoryExtension on DataCategory {
         return DataCategory.attachment;
       case 'security':
         return DataCategory.security;
+      case 'metric_bucket':
+        return DataCategory.metricBucket;
     }
     return DataCategory.unknown;
   }
@@ -47,6 +50,8 @@ extension DataCategoryExtension on DataCategory {
         return 'attachment';
       case DataCategory.security:
         return 'security';
+      case DataCategory.metricBucket:
+        return 'metric_bucket';
       case DataCategory.unknown:
         return 'unknown';
     }

dart/lib/src/transport/rate_limit.dart

@@ -2,8 +2,10 @@ import 'data_category.dart';
 
 /// `RateLimit` containing limited `DataCategory` and duration in milliseconds.
 class RateLimit {
-  RateLimit(this.category, this.duration);
+  RateLimit(this.category, this.duration, {List<String>? namespaces})
+      : namespaces = (namespaces?..removeWhere((e) => e.isEmpty)) ?? [];
 
   final DataCategory category;
   final Duration duration;
+  final List<String> namespaces;
 }

dart/lib/src/transport/rate_limit_parser.dart

@@ -14,6 +14,7 @@ class RateLimitParser {
     if (rateLimitHeader == null) {
       return [];
     }
+    // example: 2700:metric_bucket:organization:quota_exceeded:custom,...
     final rateLimits = <RateLimit>[];
     final rateLimitValues = rateLimitHeader.toLowerCase().split(',');
     for (final rateLimitValue in rateLimitValues) {
@@ -30,7 +31,17 @@ class RateLimitParser {
         final categoryValues = allCategories.split(';');
         for (final categoryValue in categoryValues) {
           final category = DataCategoryExtension.fromStringValue(categoryValue);
-          if (category != DataCategory.unknown) {
+          // Metric buckets rate limit can have namespaces
+          if (category == DataCategory.metricBucket) {
+            final namespaces = durationAndCategories.length > 4
+                ? durationAndCategories[4]
+                : null;
+            rateLimits.add(RateLimit(
+              category,
+              duration,
+              namespaces: namespaces?.trim().split(','),
+            ));
+          } else if (category != DataCategory.unknown) {
             rateLimits.add(RateLimit(category, duration));
           }
         }

dart/lib/src/transport/rate_limiter.dart

@@ -64,6 +64,11 @@ class RateLimiter {
     }
 
     for (final rateLimit in rateLimits) {
+      if (rateLimit.category == DataCategory.metricBucket &&
+          rateLimit.namespaces.isNotEmpty &&
+          !rateLimit.namespaces.contains('custom')) {
+        continue;
+      }
       _applyRetryAfterOnlyIfLonger(
         rateLimit.category,
         DateTime.fromMillisecondsSinceEpoch(
@@ -111,6 +116,10 @@ class RateLimiter {
         return DataCategory.attachment;
       case 'transaction':
         return DataCategory.transaction;
+      // The envelope item type used for metrics is statsd,
+      // whereas the client report category is metric_bucket
+      case 'statsd':
+        return DataCategory.metricBucket;
       default:
         return DataCategory.unknown;
     }

dart/test/metrics/metric_test.dart

@@ -58,10 +58,50 @@ void main() {
     test('encode CounterMetric', () async {
       final int bucketKey = 10;
       final expectedStatsd =
-          'key_metric_@hour:2.1|c|#tag1:tag value 1,key_2:@13/-d_s|T10';
+          'key_metric_@hour:2.1|c|#tag1:tag\\u{2c} value 1,key2:&@"13/-d_s|T10';
       final actualStatsd = fixture.counterMetric.encodeToStatsd(bucketKey);
       expect(actualStatsd, expectedStatsd);
     });
+
+    test('sanitize name', () async {
+      final metric = Metric.fromType(
+        type: MetricType.counter,
+        value: 2.1,
+        key: 'key£ - @# metric!',
+        unit: DurationSentryMeasurementUnit.day,
+        tags: {},
+      );
+
+      final expectedStatsd = 'key_-_metric_@day:2.1|c|T10';
+      expect(metric.encodeToStatsd(10), expectedStatsd);
+    });
+
+    test('sanitize unit', () async {
+      final metric = Metric.fromType(
+        type: MetricType.counter,
+        value: 2.1,
+        key: 'key',
+        unit: CustomSentryMeasurementUnit('weird-measurement name!'),
+        tags: {},
+      );
+
+      final expectedStatsd = 'key@weirdmeasurementname:2.1|c|T10';
+      expect(metric.encodeToStatsd(10), expectedStatsd);
+    });
+
+    test('sanitize tags', () async {
+      final metric = Metric.fromType(
+        type: MetricType.counter,
+        value: 2.1,
+        key: 'key',
+        unit: DurationSentryMeasurementUnit.day,
+        tags: {'tag1': 'tag, value 1', 'key 2': '&@"13/-d_s'},
+      );
+
+      final expectedStatsd =
+          'key@day:2.1|c|#tag1:tag\\u{2c} value 1,key2:&@"13/-d_s|T10';
+      expect(metric.encodeToStatsd(10), expectedStatsd);
+    });
   });
 
   group('getCompositeKey', () {

dart/test/protocol/rate_limit_parser_test.dart

@@ -75,6 +75,45 @@ void main() {
       expect(sut[0].duration.inMilliseconds,
           RateLimitParser.httpRetryAfterDefaultDelay.inMilliseconds);
     });
+
+    test('do not parse namespaces if not metric_bucket', () {
+      final sut =
+          RateLimitParser('1:transaction:organization:quota_exceeded:custom')
+              .parseRateLimitHeader();
+
+      expect(sut.length, 1);
+      expect(sut[0].category, DataCategory.transaction);
+      expect(sut[0].namespaces, isEmpty);
+    });
+
+    test('parse namespaces on metric_bucket', () {
+      final sut =
+          RateLimitParser('1:metric_bucket:organization:quota_exceeded:custom')
+              .parseRateLimitHeader();
+
+      expect(sut.length, 1);
+      expect(sut[0].category, DataCategory.metricBucket);
+      expect(sut[0].namespaces, isNotEmpty);
+      expect(sut[0].namespaces.first, 'custom');
+    });
+
+    test('parse empty namespaces on metric_bucket', () {
+      final sut =
+          RateLimitParser('1:metric_bucket:organization:quota_exceeded:')
+              .parseRateLimitHeader();
+
+      expect(sut.length, 1);
+      expect(sut[0].category, DataCategory.metricBucket);
+      expect(sut[0].namespaces, isEmpty);
+    });
+
+    test('parse missing namespaces on metric_bucket', () {
+      final sut = RateLimitParser('1:metric_bucket').parseRateLimitHeader();
+
+      expect(sut.length, 1);
+      expect(sut[0].category, DataCategory.metricBucket);
+      expect(sut[0].namespaces, isEmpty);
+    });
   });
 
   group('parseRetryAfterHeader', () {

dart/test/protocol/rate_limiter_test.dart

@@ -228,6 +228,118 @@ void main() {
     expect(fixture.mockRecorder.category, DataCategory.transaction);
     expect(fixture.mockRecorder.reason, DiscardReason.rateLimitBackoff);
   });
+
+  test('dropping of metrics recorded', () {
+    final rateLimiter = fixture.getSut();
+
+    final metricsItem = SentryEnvelopeItem.fromMetrics({});
+    final eventEnvelope = SentryEnvelope(
+      SentryEnvelopeHeader.newEventId(),
+      [metricsItem],
+    );
+
+    rateLimiter.updateRetryAfterLimits(
+        '1:metric_bucket:key, 5:metric_bucket:organization', null, 1);
+
+    final result = rateLimiter.filter(eventEnvelope);
+    expect(result, isNull);
+
+    expect(fixture.mockRecorder.category, DataCategory.metricBucket);
+    expect(fixture.mockRecorder.reason, DiscardReason.rateLimitBackoff);
+  });
+
+  group('apply rateLimit', () {
+    test('error', () {
+      final rateLimiter = fixture.getSut();
+      fixture.dateTimeToReturn = 0;
+
+      final eventItem = SentryEnvelopeItem.fromEvent(SentryEvent());
+      final envelope = SentryEnvelope(
+        SentryEnvelopeHeader.newEventId(),
+        [eventItem],
+      );
+
+      rateLimiter.updateRetryAfterLimits(
+          '1:error:key, 5:error:organization', null, 1);
+
+      expect(rateLimiter.filter(envelope), isNull);
+    });
+
+    test('transaction', () {
+      final rateLimiter = fixture.getSut();
+      fixture.dateTimeToReturn = 0;
+
+      final transaction = fixture.getTransaction();
+      final eventItem = SentryEnvelopeItem.fromTransaction(transaction);
+      final envelope = SentryEnvelope(
+        SentryEnvelopeHeader.newEventId(),
+        [eventItem],
+      );
+
+      rateLimiter.updateRetryAfterLimits(
+          '1:transaction:key, 5:transaction:organization', null, 1);
+
+      final result = rateLimiter.filter(envelope);
+      expect(result, isNull);
+    });
+
+    test('metrics', () {
+      final rateLimiter = fixture.getSut();
+      fixture.dateTimeToReturn = 0;
+
+      final metricsItem = SentryEnvelopeItem.fromMetrics({});
+      final envelope = SentryEnvelope(
+        SentryEnvelopeHeader.newEventId(),
+        [metricsItem],
+      );
+
+      rateLimiter.updateRetryAfterLimits(
+          '1:metric_bucket:key, 5:metric_bucket:organization', null, 1);
+
+      final result = rateLimiter.filter(envelope);
+      expect(result, isNull);
+    });
+
+    test('metrics with empty namespaces', () {
+      final rateLimiter = fixture.getSut();
+      fixture.dateTimeToReturn = 0;
+
+      final eventItem = SentryEnvelopeItem.fromEvent(SentryEvent());
+      final metricsItem = SentryEnvelopeItem.fromMetrics({});
+      final envelope = SentryEnvelope(
+        SentryEnvelopeHeader.newEventId(),
+        [eventItem, metricsItem],
+      );
+
+      rateLimiter.updateRetryAfterLimits(
+          '10:metric_bucket:key:quota_exceeded:', null, 1);
+
+      final result = rateLimiter.filter(envelope);
+      expect(result, isNotNull);
+      expect(result!.items.length, 1);
+      expect(result.items.first.header.type, 'event');
+    });
+
+    test('metrics with custom namespace', () {
+      final rateLimiter = fixture.getSut();
+      fixture.dateTimeToReturn = 0;
+
+      final eventItem = SentryEnvelopeItem.fromEvent(SentryEvent());
+      final metricsItem = SentryEnvelopeItem.fromMetrics({});
+      final envelope = SentryEnvelope(
+        SentryEnvelopeHeader.newEventId(),
+        [eventItem, metricsItem],
+      );
+
+      rateLimiter.updateRetryAfterLimits(
+          '10:metric_bucket:key:quota_exceeded:custom', null, 1);
+
+      final result = rateLimiter.filter(envelope);
+      expect(result, isNotNull);
+      expect(result!.items.length, 1);
+      expect(result.items.first.header.type, 'event');
+    });
+  });
 }
 
 class Fixture {