ref: Use average color for redact masking (getsentry#3877)

Use the text color or the average color of the image as the redaction mask.

Co-authored-by: Philipp Hofmann <[email protected]>

Samples/iOS-Swift/iOS-Swift/AppDelegate.swift

@@ -25,7 +25,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
             options.debug = true
 
             if #available(iOS 16.0, *) {
-                options.experimental.sessionReplay = SentryReplayOptions(sessionSampleRate: 0, errorSampleRate: 1, redactAllText: true, redactAllImages: true)
+                options.experimental.sessionReplay = SentryReplayOptions(sessionSampleRate: 1, errorSampleRate: 1, redactAllText: true, redactAllImages: true)
             }
 
             if #available(iOS 15.0, *) {

Sources/Sentry/SentrySessionReplay.m

@@ -35,7 +35,7 @@ @implementation SentrySessionReplay {
     NSDate *_sessionStart;
     NSMutableArray<UIImage *> *imageCollection;
     SentryReplayOptions *_replayOptions;
-    SentryOnDemandReplay *_replayMaker;
+    id<SentryReplayVideoMaker> _replayMaker;
     SentryDisplayLinkWrapper *_displayLink;
     SentryCurrentDateProvider *_dateProvider;
     id<SentryRandom> _sentryRandom;
@@ -48,7 +48,7 @@ @implementation SentrySessionReplay {
 - (instancetype)initWithSettings:(SentryReplayOptions *)replayOptions
                 replayFolderPath:(NSURL *)folderPath
               screenshotProvider:(id<SentryViewScreenshotProvider>)screenshotProvider
-                     replayMaker:(id<SentryReplayMaker>)replayMaker
+                     replayMaker:(id<SentryReplayVideoMaker>)replayMaker
                     dateProvider:(SentryCurrentDateProvider *)dateProvider
                           random:(id<SentryRandom>)random
               displayLinkWrapper:(SentryDisplayLinkWrapper *)displayLinkWrapper;
@@ -242,6 +242,7 @@ - (void)createAndCapture:(NSURL *)videoUrl
                 duration:(NSTimeInterval)duration
                startedAt:(NSDate *)start
 {
+    __weak SentrySessionReplay *weakSelf = self;
     [_replayMaker
         createVideoWithDuration:duration
                       beginning:start
@@ -251,17 +252,22 @@ - (void)createAndCapture:(NSURL *)videoUrl
                          if (error != nil) {
                              SENTRY_LOG_ERROR(@"Could not create replay video - %@", error);
                          } else {
-                             [self captureSegment:self->_currentSegmentId++
-                                            video:videoInfo
-                                         replayId:self->_sessionReplayId
-                                       replayType:kSentryReplayTypeSession];
-
-                             [self->_replayMaker releaseFramesUntil:videoInfo.end];
-                             self->_videoSegmentStart = nil;
+                             [weakSelf newSegmentAvailable:videoInfo];
                          }
                      }];
 }
 
+- (void)newSegmentAvailable:(SentryVideoInfo *)videoInfo
+{
+    [self captureSegment:self->_currentSegmentId++
+                   video:videoInfo
+                replayId:self->_sessionReplayId
+              replayType:kSentryReplayTypeSession];
+
+    [_replayMaker releaseFramesUntil:videoInfo.end];
+    _videoSegmentStart = nil;
+}
+
 - (void)captureSegment:(NSInteger)segment
                  video:(SentryVideoInfo *)videoInfo
               replayId:(SentryId *)replayid
@@ -306,11 +312,16 @@ - (void)takeScreenshot
         _processingScreenshot = YES;
     }
 
-    UIImage *screenshot = [_screenshotProvider imageWithView:_rootView options:_replayOptions];
+    __weak SentrySessionReplay *weakSelf = self;
+    [_screenshotProvider imageWithView:_rootView
+                               options:_replayOptions
+                            onComplete:^(UIImage *screenshot) { [weakSelf newImage:screenshot]; }];
+}
 
+- (void)newImage:(UIImage *)image
+{
     _processingScreenshot = NO;
-
-    [self->_replayMaker addFrameAsyncWithImage:screenshot];
+    [_replayMaker addFrameAsyncWithImage:image];
 }
 
 @end

Sources/Sentry/SentrySessionReplayIntegration.m

@@ -28,17 +28,6 @@
 - (void)newSceneActivate;
 @end
 
-API_AVAILABLE(ios(16.0), tvos(16.0))
-@interface
-SentryViewPhotographer (SentryViewScreenshotProvider) <SentryViewScreenshotProvider>
-@end
-
-API_AVAILABLE(ios(16.0), tvos(16.0))
-@interface
-SentryOnDemandReplay (SentryReplayMaker) <SentryReplayMaker>
-
-@end
-
 @implementation SentrySessionReplayIntegration {
     BOOL _startedAsFullSession;
     SentryReplayOptions *_replayOptions;

Sources/Sentry/include/SentryPrivate.h

@@ -11,4 +11,3 @@
 
 // Headers that also import SentryDefines should be at the end of this list
 // otherwise it wont compile
-#import "SentryCoreGraphicsHelper.h"

Sources/Sentry/include/SentrySessionReplay.h

@@ -13,35 +13,19 @@
 
 @protocol SentryRandom;
 @protocol SentryRedactOptions;
+@protocol SentryViewScreenshotProvider;
+@protocol SentryReplayVideoMaker;
 
 NS_ASSUME_NONNULL_BEGIN
 
-@protocol SentryReplayMaker <NSObject>
-
-- (void)addFrameAsyncWithImage:(UIImage *)image;
-- (void)releaseFramesUntil:(NSDate *)date;
-- (BOOL)createVideoWithDuration:(NSTimeInterval)duration
-                      beginning:(NSDate *)beginning
-                  outputFileURL:(NSURL *)outputFileURL
-                          error:(NSError *_Nullable *_Nullable)error
-                     completion:
-                         (void (^)(SentryVideoInfo *_Nullable, NSError *_Nullable))completion;
-
-@end
-
-@protocol SentryViewScreenshotProvider <NSObject>
-- (UIImage *)imageWithView:(UIView *)view options:(id<SentryRedactOptions>)options;
-@end
-
-API_AVAILABLE(ios(16.0), tvos(16.0))
 @interface SentrySessionReplay : NSObject
 
 @property (nonatomic, strong, readonly) SentryId *sessionReplayId;
 
 - (instancetype)initWithSettings:(SentryReplayOptions *)replayOptions
                 replayFolderPath:(NSURL *)folderPath
               screenshotProvider:(id<SentryViewScreenshotProvider>)photographer
-                     replayMaker:(id<SentryReplayMaker>)replayMaker
+                     replayMaker:(id<SentryReplayVideoMaker>)replayMaker
                     dateProvider:(SentryCurrentDateProvider *)dateProvider
                           random:(id<SentryRandom>)random
               displayLinkWrapper:(SentryDisplayLinkWrapper *)displayLinkWrapper;

Sources/Swift/Integrations/SessionReplay/SentryOnDemandReplay.swift

@@ -22,7 +22,7 @@ enum SentryOnDemandReplayError: Error {
 }
 
 @objcMembers
-class SentryOnDemandReplay: NSObject {
+class SentryOnDemandReplay: NSObject, SentryReplayVideoMaker {
     private let _outputPath: String
     private var _currentPixelBuffer: SentryPixelBuffer?
     private var _totalFrames = 0

Sources/Swift/Integrations/SessionReplay/SentryReplayVideoMaker.swift

@@ -0,0 +1,11 @@
+#if canImport(UIKit)
+import Foundation
+import UIKit
+
+@objc
+protocol SentryReplayVideoMaker: NSObjectProtocol {
+    func addFrameAsync(image: UIImage)
+    func releaseFramesUntil(_ date: Date)
+    func createVideoWith(duration: TimeInterval, beginning: Date, outputFileURL: URL, completion: @escaping (SentryVideoInfo?, Error?) -> Void) throws
+}
+#endif

Sources/Swift/Tools/SentryViewPhotographer.swift

@@ -1,123 +1,47 @@
 #if canImport(UIKit) && !SENTRY_NO_UIKIT
 #if os(iOS) || os(tvOS)
 
-@_implementationOnly import _SentryPrivate
 import CoreGraphics
 import Foundation
 import UIKit
 
-@available(iOS, introduced: 16.0)
-@available(tvOS, introduced: 16.0)
 @objcMembers
-class SentryViewPhotographer: NSObject {
-
-    //This is a list of UIView subclasses that will be ignored during redact process
-    private var ignoreClasses: [AnyClass] = []
-    //This is a list of UIView subclasses that need to be redacted from screenshot
-    private var redactClasses: [AnyClass] = []
+class SentryViewPhotographer: NSObject, SentryViewScreenshotProvider {
 
     static let shared = SentryViewPhotographer()
 
-    override init() {
-#if os(iOS)
-        ignoreClasses = [  UISlider.self, UISwitch.self ]
-#endif // os(iOS)
-        redactClasses = [ UILabel.self, UITextView.self, UITextField.self ] + [
-            "_TtCOCV7SwiftUI11DisplayList11ViewUpdater8Platform13CGDrawingView",
-            "_TtC7SwiftUIP33_A34643117F00277B93DEBAB70EC0697122_UIShapeHitTestingView",
-            "SwiftUI._UIGraphicsView", "SwiftUI.ImageLayer"
-        ].compactMap { NSClassFromString($0) }
-    }
-
-    @objc(imageWithView:options:)
-    func image(view: UIView, options: SentryRedactOptions) -> UIImage? {
-        UIGraphicsBeginImageContextWithOptions(view.bounds.size, true, 0)
+    //This is a list of UIView subclasses that will be ignored during redact process
+    private var redactBuilder = UIRedactBuilder()
 
-        defer {
-            UIGraphicsEndImageContext()
+    func image(view: UIView, options: SentryRedactOptions, onComplete: @escaping ScreenshotCallback ) {
+        let image = UIGraphicsImageRenderer(size: view.bounds.size).image { _ in
+            view.drawHierarchy(in: view.bounds, afterScreenUpdates: false)
         }
 
-        guard let currentContext = UIGraphicsGetCurrentContext() else { return nil }
-
-        view.layer.render(in: currentContext)
-        self.mask(view: view, context: currentContext, options: options)
-
-        guard let screenshot = UIGraphicsGetImageFromCurrentImageContext() else { return nil }
-        return screenshot
+        let redact = redactBuilder.redactRegionsFor(view: view, options: options)
+        let imageSize = view.bounds.size
+        DispatchQueue.global().async {
+            let screenshot = UIGraphicsImageRenderer(size: imageSize, format: .init(for: .init(displayScale: 1))).image { context in
+                context.cgContext.interpolationQuality = .none
+                image.draw(at: .zero)
+
+                for region in redact {
+                    (region.color ?? UIImageHelper.averageColor(of: context.currentImage, at: region.rect)).setFill()
+                    context.fill(region.rect)
+                }
+            }
+            onComplete(screenshot)
+        }
     }
-
+    
     @objc(addIgnoreClasses:)
     func addIgnoreClasses(classes: [AnyClass]) {
-        ignoreClasses += classes
+        redactBuilder.ignoreClasses += classes
     }
 
     @objc(addRedactClasses:)
     func addRedactClasses(classes: [AnyClass]) {
-        redactClasses += classes
-    }
-
-    private func mask(view: UIView, context: CGContext, options: SentryRedactOptions?) {
-        UIColor.black.setFill()
-        let maskPath = self.buildPath(view: view,
-                                      path: CGMutablePath(),
-                                      area: view.frame,
-                                      redactText: options?.redactAllText ?? true,
-                                      redactImage: options?.redactAllImages ?? true)
-        context.addPath(maskPath)
-        context.fillPath()
-    }
-
-    private func shouldIgnore(view: UIView) -> Bool {
-        ignoreClasses.contains { view.isKind(of: $0) }
-    }
-
-    private func shouldRedact(view: UIView) -> Bool {
-       return redactClasses.contains { view.isKind(of: $0) }
-    }
-
-    private func shouldRedact(imageView: UIImageView) -> Bool {
-        // Checking the size is to avoid redact gradient backgroud that
-        // are usually small lines repeating
-        guard let image = imageView.image, image.size.width > 10 && image.size.height > 10  else { return false }
-        return image.imageAsset?.value(forKey: "_containingBundle") == nil
-    }
-
-    private func buildPath(view: UIView, path: CGMutablePath, area: CGRect, redactText: Bool, redactImage: Bool) -> CGMutablePath {
-        let rectInWindow = view.convert(view.bounds, to: nil)
-
-        if (!redactImage && !redactText) || !area.intersects(rectInWindow) || view.isHidden || view.alpha == 0 {
-            return path
-        }
-
-        var result = path
-
-        let ignore = shouldIgnore(view: view)
-
-        let redact: Bool = {
-            if redactImage, let imageView = view as? UIImageView {
-                return shouldRedact(imageView: imageView)
-            }
-            return redactText && shouldRedact(view: view)
-        }()
-
-        if !ignore && redact {
-            result.addRect(rectInWindow)
-            return result
-        } else if isOpaqueOrHasBackground(view) {
-            result = SentryCoreGraphicsHelper.excludeRect(rectInWindow, from: result).takeRetainedValue()
-        }
-
-        if !ignore {
-            for subview in view.subviews {
-                result = buildPath(view: subview, path: path, area: area, redactText: redactText, redactImage: redactImage)
-            }
-        }
-
-        return result
-    }
-
-    private func isOpaqueOrHasBackground(_ view: UIView) -> Bool {
-        return view.isOpaque || (view.backgroundColor != nil && (view.backgroundColor?.cgColor.alpha ?? 0) > 0.9)
+        redactBuilder.redactClasses += classes
     }
 }
 

Sources/Swift/Tools/SentryViewScreenshotProvider.swift

@@ -0,0 +1,13 @@
+#if canImport(UIKit) && !SENTRY_NO_UIKIT
+#if os(iOS) || os(tvOS)
+import Foundation
+import UIKit
+
+typealias ScreenshotCallback = (UIImage) -> Void
+
+@objc
+protocol SentryViewScreenshotProvider: NSObjectProtocol {
+    func image(view: UIView, options: SentryRedactOptions, onComplete: @escaping ScreenshotCallback)
+}
+#endif
+#endif

Sources/Swift/Tools/UIImageHelper.swift

@@ -0,0 +1,36 @@
+#if canImport(UIKit) && !SENTRY_NO_UIKIT
+#if os(iOS) || os(tvOS)
+
+import Foundation
+import UIKit
+
+final class UIImageHelper {
+    private init() { }
+
+    static func averageColor(of image: UIImage, at region: CGRect) -> UIColor {
+        let scaledRegion = region.applying(CGAffineTransform(scaleX: image.scale, y: image.scale))
+        guard let croppedImage = image.cgImage?.cropping(to: scaledRegion), let colorSpace = croppedImage.colorSpace else {
+            return .black
+        }
+
+        let bitmapInfo: UInt32 = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue
+
+        guard let context = CGContext(data: nil, width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 4, space: colorSpace, bitmapInfo: bitmapInfo) else { return .black }
+        context.interpolationQuality = .high
+        context.draw(croppedImage, in: CGRect(x: 0, y: 0, width: 1, height: 1))
+        guard let pixelBuffer = context.data else { return .black }
+
+        let data = pixelBuffer.bindMemory(to: UInt8.self, capacity: 4)
+
+        let blue = CGFloat(data[0]) / 255.0
+        let green = CGFloat(data[1]) / 255.0
+        let red = CGFloat(data[2]) / 255.0
+        let alpha = CGFloat(data[3]) / 255.0
+
+        return UIColor(red: red, green: green, blue: blue, alpha: alpha)
+    }
+
+}
+
+#endif
+#endif