feat(Session Replay): Add breadcrumbs to session replay (#4002)

Converting session breadcrumbs to replay breadcrumbs

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- Add breadcrumbs to session replay (#4002)
 - Add start time to network request breadcrumbs (#4008)
 - Add C++ exception support for `__cxa_rethrow` (#3996)
 - Add beforeCaptureScreenshot callback (#4016)

Sources/Sentry/SentryLevelHelper.m

@@ -0,0 +1,11 @@
+#import "SentryLevelHelper.h"
+#import "SentryBreadcrumb+Private.h"
+
+@implementation SentryLevelHelper
+
++ (NSUInteger)breadcrumbLevel:(SentryBreadcrumb *)breadcrumb
+{
+    return breadcrumb.level;
+}
+
+@end

Sources/Sentry/SentryScope.m

@@ -30,11 +30,6 @@
  */
 @property (atomic, strong) NSMutableDictionary<NSString *, id> *extraDictionary;
 
-/**
- * Contains the breadcrumbs which will be sent with the event
- */
-@property (atomic, strong) NSMutableArray<SentryBreadcrumb *> *breadcrumbArray;
-
 /**
  * This distribution of the application.
  */

Sources/Sentry/SentrySessionReplay.m

@@ -1,5 +1,6 @@
 #import "SentrySessionReplay.h"
 #import "SentryAttachment+Private.h"
+#import "SentryBreadcrumb+Private.h"
 #import "SentryDependencyContainer.h"
 #import "SentryDisplayLinkWrapper.h"
 #import "SentryEnvelopeItemType.h"
@@ -42,6 +43,7 @@ @implementation SentrySessionReplay {
     int _currentSegmentId;
     BOOL _processingScreenshot;
     BOOL _reachedMaximumDuration;
+    SentryReplayBreadcrumbConverter *_breadcrumbConverter;
 }
 
 - (instancetype)initWithSettings:(SentryReplayOptions *)replayOptions
@@ -62,6 +64,7 @@ - (instancetype)initWithSettings:(SentryReplayOptions *)replayOptions
         _urlToCache = folderPath;
         _replayMaker = replayMaker;
         _reachedMaximumDuration = NO;
+        _breadcrumbConverter = [[SentryReplayBreadcrumbConverter alloc] init];
     }
     return self;
 }
@@ -286,6 +289,14 @@ - (void)captureSegment:(NSInteger)segment
     replayEvent.segmentId = segment;
     replayEvent.timestamp = videoInfo.end;
 
+    __block NSArray<SentryRRWebEvent *> *events;
+
+    [SentrySDK.currentHub configureScope:^(SentryScope *_Nonnull scope) {
+        events = [self->_breadcrumbConverter convertWithBreadcrumbs:scope.breadcrumbArray
+                                                               from:videoInfo.start
+                                                              until:videoInfo.end];
+    }];
+
     SentryReplayRecording *recording =
         [[SentryReplayRecording alloc] initWithSegmentId:replayEvent.segmentId
                                                     size:videoInfo.fileSize
@@ -294,7 +305,8 @@ - (void)captureSegment:(NSInteger)segment
                                               frameCount:videoInfo.frameCount
                                                frameRate:videoInfo.frameRate
                                                   height:videoInfo.height
-                                                   width:videoInfo.width];
+                                                   width:videoInfo.width
+                                             extraEvents:events];
 
     [SentrySDK.currentHub captureReplayEvent:replayEvent
                              replayRecording:recording

Sources/Sentry/include/SentryLevelHelper.h

@@ -0,0 +1,16 @@
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class SentryBreadcrumb;
+
+/**
+ * This is a workaround to access SentryLevel value from swift
+ */
+@interface SentryLevelHelper : NSObject
+
++ (NSUInteger)breadcrumbLevel:(SentryBreadcrumb *)breadcrumb;
+
+@end
+
+NS_ASSUME_NONNULL_END

Sources/Sentry/include/SentryPrivate.h

@@ -11,4 +11,5 @@
 // Headers that also import SentryDefines should be at the end of this list
 // otherwise it wont compile
 #import "SentryDateUtil.h"
+#import "SentryLevelHelper.h"
 #import "SentrySdkInfo.h"

Sources/Sentry/include/SentryScope+Private.h

@@ -23,6 +23,11 @@ SentryScope ()
 
 @property (atomic, strong) SentryPropagationContext *propagationContext;
 
+/**
+ * Contains the breadcrumbs which will be sent with the event
+ */
+@property (atomic, strong) NSMutableArray<SentryBreadcrumb *> *breadcrumbArray;
+
 /**
  * used to add values in event context.
  */

Sources/Swift/Extensions/StringExtensions.swift

@@ -37,4 +37,20 @@ extension String {
         let end = index(startIndex, offsetBy: range.upperBound)
         return self[startIndex..<end]
     }
+
+    func snakeToCamelCase() -> String {
+        var result = ""
+
+        var toUpper = false
+        for char in self {
+            if char == "_" {
+                toUpper = true
+            } else {
+                result.append(toUpper ? char.uppercased() : String(char))
+                toUpper = false
+            }
+        }
+
+        return result
+    }
 }

Sources/Swift/Integrations/SessionReplay/RRWeb/SentryRRWebBreadcrumbEvent.swift

@@ -1,10 +1,10 @@
+@_implementationOnly import _SentryPrivate
 import Foundation
 
-@objcMembers
 class SentryRRWebBreadcrumbEvent: SentryRRWebCustomEvent {
-    init(timestamp: Date, category: String, message: String?, level: SentryLevel, data: [String: Any]?) {
+    init(timestamp: Date, category: String, message: String? = nil, level: SentryLevel = .none, data: [String: Any]? = nil) {
 
-        var payload: [String: Any] = ["type": "default", "category": category, "level": level.rawValue ]
+        var payload: [String: Any] = ["type": "default", "category": category, "level": level.rawValue, "timestamp": timestamp.timeIntervalSince1970 ]
 
         if let message = message {
             payload["message"] = message

Sources/Swift/Integrations/SessionReplay/RRWeb/SentryRRWebSpanEvent.swift

@@ -0,0 +1,17 @@
+@_implementationOnly import _SentryPrivate
+import Foundation
+
+@objc class SentryRRWebSpanEvent: SentryRRWebCustomEvent {
+
+    init(timestamp: Date, endTimestamp: Date, operation: String, description: String, data: [String: Any]) {
+        super.init(timestamp: timestamp, tag: "performanceSpan", payload:
+                    [
+                        "op": operation,
+                        "description": description,
+                        "startTimestamp": timestamp.timeIntervalSince1970,
+                        "endTimestamp": endTimestamp.timeIntervalSince1970,
+                        "data": data
+                    ]
+        )
+    }
+}

Sources/Swift/Integrations/SessionReplay/RRWeb/SentryRRWebVideoEvent.swift

@@ -6,7 +6,7 @@ class SentryRRWebVideoEvent: SentryRRWebCustomEvent {
     init(timestamp: Date, segmentId: Int, size: Int, duration: TimeInterval, encoding: String, container: String, height: Int, width: Int, frameCount: Int, frameRateType: String, frameRate: Int, left: Int, top: Int) {
 
         super.init(timestamp: timestamp, tag: "video", payload: [
-            "timestamp": SentryDateUtil.millisecondsSince1970(timestamp),
+            "timestamp": timestamp.timeIntervalSince1970,
             "segmentId": segmentId,
             "size": size,
             "duration": duration,

Sources/Swift/Integrations/SessionReplay/SentryReplayBreadcrumbConverter.swift

@@ -0,0 +1,92 @@
+@_implementationOnly import _SentryPrivate
+import Foundation
+
+@objcMembers
+class SentryReplayBreadcrumbConverter: NSObject {
+
+    private let supportedNetworkData = Set<String>([
+        "status_code",
+        "method",
+        "response_body_size",
+        "request_body_size",
+        "http.query",
+        "http.fragment"]
+    )
+
+    func convert(breadcrumbs: [Breadcrumb], from: Date, until: Date) -> [SentryRRWebEvent] {
+        breadcrumbs.filter {
+            guard let timestamp = $0.timestamp else { return false }
+            return timestamp >= from && timestamp <= until
+        }
+        .compactMap { convert(from: $0) }
+    }
+
+    /**
+     * This function will convert the SDK breadcrumbs to session replay breadcrumbs in a format that the front-end understands.
+     * Any deviation in the information will cause the breadcrumb or the information itself to be discarded
+     * in order to avoid unknown behavior in the front-end.
+     */
+    private func convert(from breadcrumb: Breadcrumb) -> SentryRRWebEvent? {
+        guard let timestamp = breadcrumb.timestamp else { return nil }
+        if breadcrumb.category == "http" {
+            return networkSpan(breadcrumb)
+        } else if breadcrumb.type == "navigation" {
+            return navigationBreadcrumb(breadcrumb)
+        } else if breadcrumb.category == "touch" {
+            return SentryRRWebBreadcrumbEvent(timestamp: timestamp, category: "ui.tap", message: breadcrumb.message)
+        } else if breadcrumb.type == "connectivity" && breadcrumb.category == "device.connectivity" {
+            guard let networkType = breadcrumb.data?["connectivity"] as? String, !networkType.isEmpty  else { return nil }
+            return SentryRRWebBreadcrumbEvent(timestamp: timestamp, category: "device.connectivity", data: ["state": networkType])
+        } else if let action = breadcrumb.data?["action"] as? String, action == "BATTERY_STATE_CHANGE" {
+            var data = breadcrumb.data?.filter({ item in item.key == "level" || item.key == "plugged" }) ?? [:]
+
+            data["charging"] = data["plugged"]
+            data["plugged"] = nil
+
+            return SentryRRWebBreadcrumbEvent(timestamp: timestamp,
+                                              category: "device.battery",
+                                              data: data)
+        }
+
+        let level = getLevel(breadcrumb: breadcrumb)
+        return SentryRRWebBreadcrumbEvent(timestamp: timestamp, category: breadcrumb.category, message: breadcrumb.message, level: level, data: breadcrumb.data)
+    }
+
+    private func navigationBreadcrumb(_ breadcrumb: Breadcrumb) -> SentryRRWebBreadcrumbEvent? {
+        guard let timestamp = breadcrumb.timestamp else { return nil }
+
+        if breadcrumb.category == "app.lifecycle" {
+            guard let state = breadcrumb.data?["state"] else { return nil }
+            return SentryRRWebBreadcrumbEvent(timestamp: timestamp, category: "app.\(state)")
+        } else if let position = breadcrumb.data?["position"] as? String, breadcrumb.category == "device.orientation" {
+            return SentryRRWebBreadcrumbEvent(timestamp: timestamp, category: "device.orientation", data: ["position": position])
+        } else {
+            if let to = breadcrumb.data?["screen"] as? String {
+                return SentryRRWebBreadcrumbEvent(timestamp: timestamp, category: "navigation", message: to, data: ["to": to])
+            } else {
+                return nil
+            }
+        }
+    }
+
+    private func networkSpan(_ breadcrumb: Breadcrumb) -> SentryRRWebSpanEvent? {
+        guard let timestamp = breadcrumb.timestamp,
+              let description = breadcrumb.data?["url"] as? String,
+              let startTimestamp = breadcrumb.data?["request_start"] as? Date
+        else { return nil }
+        var data = [String: Any]()
+
+        breadcrumb.data?.forEach({ (key, value) in
+            guard supportedNetworkData.contains(key) else { return }
+            let newKey = key.replacingOccurrences(of: "http.", with: "")
+            data[newKey.snakeToCamelCase()] = value
+        })
+
+        //We dont have end of the request in the breadcrumb.
+        return SentryRRWebSpanEvent(timestamp: startTimestamp, endTimestamp: timestamp, operation: "resource.http", description: description, data: data)
+    }
+
+    private func getLevel(breadcrumb: Breadcrumb) -> SentryLevel {
+        return SentryLevel(rawValue: SentryLevelHelper.breadcrumbLevel(breadcrumb)) ?? .none
+    }
+}

Sources/Swift/Integrations/SessionReplay/SentryReplayRecording.swift

@@ -10,25 +10,21 @@ class SentryReplayRecording: NSObject {
 
     let segmentId: Int
 
-    let meta: SentryRRWebMetaEvent
-    let video: SentryRRWebVideoEvent
+    let events: [SentryRRWebEvent]
 
-    init(segmentId: Int, size: Int, start: Date, duration: TimeInterval, frameCount: Int, frameRate: Int, height: Int, width: Int) {
+    init(segmentId: Int, size: Int, start: Date, duration: TimeInterval, frameCount: Int, frameRate: Int, height: Int, width: Int, extraEvents: [SentryRRWebEvent]?) {
         self.segmentId = segmentId
 
-        meta = SentryRRWebMetaEvent(timestamp: start, height: height, width: width)
-        video = SentryRRWebVideoEvent(timestamp: start, segmentId: segmentId, size: size, duration: duration, encoding: SentryReplayRecording.SentryReplayEncoding, container: SentryReplayRecording.SentryReplayContainer, height: height, width: width, frameCount: frameCount, frameRateType: SentryReplayRecording.SentryReplayFrameRateType, frameRate: frameRate, left: 0, top: 0)
+        let meta = SentryRRWebMetaEvent(timestamp: start, height: height, width: width)
+        let video = SentryRRWebVideoEvent(timestamp: start, segmentId: segmentId, size: size, duration: duration, encoding: SentryReplayRecording.SentryReplayEncoding, container: SentryReplayRecording.SentryReplayContainer, height: height, width: width, frameCount: frameCount, frameRateType: SentryReplayRecording.SentryReplayFrameRateType, frameRate: frameRate, left: 0, top: 0)
+        self.events = [meta, video] + (extraEvents ?? [])
     }
 
     func headerForReplayRecording() -> [String: Any] {
         return ["segment_id": segmentId]
     }
 
     func serialize() -> [[String: Any]] {
-        let metaInfo = meta.serialize()
-
-        let recordingInfo = video.serialize()
-
-        return [metaInfo, recordingInfo]
+        return events.map { $0.serialize() }
     }
 }

Tests/SentryTests/Extensions/StringExtensionsTests.swift

@@ -0,0 +1,11 @@
+import Foundation
+@testable import Sentry
+import XCTest
+
+class StringExtensionsTests: XCTestCase {
+    func testSnakeToCamelCase() {
+        XCTAssertEqual("name_something".snakeToCamelCase(), "nameSomething")
+        XCTAssertEqual("name_something_else".snakeToCamelCase(), "nameSomethingElse")
+        XCTAssertEqual("KEEP_CASE".snakeToCamelCase(), "KEEPCASE")
+    }
+}

Tests/SentryTests/Helper/SentrySerializationTests.swift

@@ -240,7 +240,7 @@ class SentrySerializationTests: XCTestCase {
         }
 
         let date = Date(timeIntervalSince1970: 2)
-        let recording = MockReplayRecording(segmentId: 5, size: 5_000, start: date, duration: 5_000, frameCount: 5, frameRate: 1, height: 320, width: 950)
+        let recording = MockReplayRecording(segmentId: 5, size: 5_000, start: date, duration: 5_000, frameCount: 5, frameRate: 1, height: 320, width: 950, extraEvents: [])
         let data = SentrySerialization.data(with: recording)
 
         let serialized = String(data: data, encoding: .utf8)