fix(session-replay): add masking improvements

Samples/iOS-Swift/iOS-Swift/ErrorsViewController.swift

@@ -17,6 +17,8 @@ class ErrorsViewController: UIViewController {
 
     override func viewDidAppear(_ animated: Bool) {
         super.viewDidAppear(animated)
+        print("--> ErrorsViewController.viewDidAppear")
+
         SentrySDK.reportFullyDisplayed()
 
         if ProcessInfo.processInfo.arguments.contains("--io.sentry.feedback.inject-screenshot") {

Samples/iOS-Swift/iOS-Swift/SentrySDKWrapper.swift

@@ -19,8 +19,51 @@
         options.debug = true
 
         if #available(iOS 16.0, *), enableSessionReplay {
-            options.sessionReplay = SentryReplayOptions(sessionSampleRate: 0, onErrorSampleRate: 1, maskAllText: true, maskAllImages: true)
+            options.sessionReplay = SentryReplayOptions(sessionSampleRate: 0.0, onErrorSampleRate: 1, maskAllText: true, maskAllImages: true)
             options.sessionReplay.quality = .high
+            options.sessionReplay.frameRate = 10
+            try! FileManager.default.removeItem(atPath: "/tmp/workdir")
+            try! FileManager.default.createDirectory(atPath: "/tmp/workdir", withIntermediateDirectories: true, attributes: nil)
+            var previousEncodedViewData: Data?
+            var counter = 0
+            options.sessionReplay.onNewFrame = { _, viewHiearchy, redactRegions, renderedViewImage, maskedViewImage  in
+                guard TransactionsViewController.isTransitioning else { return }
+                do {
+                    let encoder = JSONEncoder()
+                    encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+
+                    let encodedViewData = try encoder.encode(viewHiearchy)
+                    if let previousEncodedViewData = previousEncodedViewData {
+                        if encodedViewData == previousEncodedViewData {
+                            return
+                        }
+                    }
+                    previousEncodedViewData = encodedViewData
+                    if counter >= 2 {
+                        return
+                    }
+                    counter += 1
+
+                    let viewHiearchyPath = "/tmp/workdir/\(counter)-0_view.json"
+                    let regionsPath = "/tmp/workdir/\(counter)-1_regions.json"
+                    let imagePath = "/tmp/workdir/\(counter)-2_image.png"
+                    let maskedImagePath = "/tmp/workdir/\(counter)-3_masked.png"
+
+                    try encodedViewData.write(to: URL(fileURLWithPath: viewHiearchyPath))
+
+                    let encodedRegionsData = try encoder.encode(redactRegions)
+                    try encodedRegionsData.write(to: URL(fileURLWithPath: regionsPath))
+
+                    let encodedImage = renderedViewImage.pngData()
+                    try encodedImage?.write(to: URL(fileURLWithPath: imagePath))
+
+                    let encodedMaskedImage = maskedViewImage.pngData()
+                    try encodedMaskedImage?.write(to: URL(fileURLWithPath: maskedImagePath))
+
+                } catch {
+                    print("Could not encode redact regions. Error: \(error)")
+                }
+            }
         }
 
         if #available(iOS 15.0, *), enableMetricKit {
@@ -60,7 +103,7 @@
         options.enableNetworkBreadcrumbs = enableNetworkBreadcrumbs
         options.enableSwizzling = enableSwizzling
         options.enableCrashHandler = enableCrashHandling
         options.enableTracing = enableTracing
         options.enablePersistingTracesWhenCrashing = true
         options.attachScreenshot = enableAttachScreenshot
         options.attachViewHierarchy = enableAttachViewHierarchy

Samples/iOS-Swift/iOS-Swift/Tools/DSNDisplayViewController.swift

@@ -15,13 +15,13 @@ class DSNDisplayViewController: UIViewController {
 
     override init(nibName nibNameOrNil: String?, bundle nibBundleOrNil: Bundle?) {
         super.init(nibName: nibNameOrNil, bundle: nibBundleOrNil)
-        
+
         if #available(iOS 13.0, *) {
             view.backgroundColor = .systemFill
         } else {
             view.backgroundColor = .lightGray.withAlphaComponent(0.5)
         }
-        
+
         label.numberOfLines = 0
         label.lineBreakMode = .byCharWrapping
         label.textAlignment = .center

Samples/iOS-Swift/iOS-Swift/TransactionsViewController.swift

@@ -2,29 +2,37 @@ import Sentry
 import UIKit
 
 class TransactionsViewController: UIViewController {
-
+    static var isTransitioning = false
     @IBOutlet weak var appHangFullyBlockingButton: UIButton!
 
     private let dispatchQueue = DispatchQueue(label: "ViewController", attributes: .concurrent)
     private var timer: Timer?
     @IBOutlet weak var dsnView: UIView!
-    
+
     override func viewDidLoad() {
         super.viewDidLoad()
         addDSNDisplay(self, vcview: dsnView)
         SentrySDK.reportFullyDisplayed()
     }
-
+
+    override func viewWillAppear(_ animated: Bool) {
+        super.viewWillAppear(animated)
+        print("--> TransactionsViewController.viewWillAppear")
+        TransactionsViewController.isTransitioning = true
+    }
+
     override func viewDidAppear(_ animated: Bool) {
         super.viewDidAppear(animated)
+        print("--> TransactionsViewController.viewDidAppear")
+        TransactionsViewController.isTransitioning = false
         periodicallyDoWork()
     }
-    
+
     override func viewDidDisappear(_ animated: Bool) {
         super .viewDidDisappear(animated)
         self.timer?.invalidate()
     }
-    
+
     private func periodicallyDoWork() {
 
         self.timer = Timer.scheduledTimer(withTimeInterval: 5.0, repeats: true) { _ in

Sentry.xcodeproj/project.pbxproj

@@ -815,6 +815,9 @@
 		A8F17B342902870300990B25 /* SentryHttpStatusCodeRange.m in Sources */ = {isa = PBXBuildFile; fileRef = A8F17B332902870300990B25 /* SentryHttpStatusCodeRange.m */; };
 		D4009EB22D771BC20007AF30 /* SentryFileIOTrackerSwiftHelpersTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4009EB12D771BB90007AF30 /* SentryFileIOTrackerSwiftHelpersTests.swift */; };
 		D42E48572D48DF1600D251BC /* SentryBuildAppStartSpansTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D42E48562D48DF1600D251BC /* SentryBuildAppStartSpansTests.swift */; };
+		D43546462D8C2B0B00E9C810 /* RedactRegionType.swift in Sources */ = {isa = PBXBuildFile; fileRef = D43546452D8C2B0A00E9C810 /* RedactRegionType.swift */; };
+		D43546482D8C2B2300E9C810 /* RedactRegion.swift in Sources */ = {isa = PBXBuildFile; fileRef = D43546472D8C2B2300E9C810 /* RedactRegion.swift */; };
+		D435464A2D8C32A500E9C810 /* ViewHierarchyNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = D43546492D8C32A400E9C810 /* ViewHierarchyNode.swift */; };
 		D43647F12D5CFB71001468E0 /* SentrySpanKeyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D43647F02D5CFB71001468E0 /* SentrySpanKeyTests.swift */; };
 		D43647F32D5CFBC7001468E0 /* FileManagerTracingIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D43647F22D5CFBC2001468E0 /* FileManagerTracingIntegrationTests.swift */; };
 		D43B26D62D70964C007747FD /* SentrySpanOperation.m in Sources */ = {isa = PBXBuildFile; fileRef = D43B26D52D709648007747FD /* SentrySpanOperation.m */; };
@@ -1974,6 +1977,9 @@
 		D41909942D490006002B83D0 /* SentryNSDictionarySanitize+Tests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "SentryNSDictionarySanitize+Tests.m"; sourceTree = "<group>"; };
 		D42E48562D48DF1600D251BC /* SentryBuildAppStartSpansTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryBuildAppStartSpansTests.swift; sourceTree = "<group>"; };
 		D42E48582D48FC8F00D251BC /* SentryNSDictionarySanitizeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryNSDictionarySanitizeTests.swift; sourceTree = "<group>"; };
+		D43546452D8C2B0A00E9C810 /* RedactRegionType.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RedactRegionType.swift; sourceTree = "<group>"; };
+		D43546472D8C2B2300E9C810 /* RedactRegion.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RedactRegion.swift; sourceTree = "<group>"; };
+		D43546492D8C32A400E9C810 /* ViewHierarchyNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewHierarchyNode.swift; sourceTree = "<group>"; };
 		D43647F02D5CFB71001468E0 /* SentrySpanKeyTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SentrySpanKeyTests.swift; sourceTree = "<group>"; };
 		D43647F22D5CFBC2001468E0 /* FileManagerTracingIntegrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileManagerTracingIntegrationTests.swift; sourceTree = "<group>"; };
 		D43B26D52D709648007747FD /* SentrySpanOperation.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentrySpanOperation.m; sourceTree = "<group>"; };
@@ -3857,6 +3863,17 @@
 			path = ViewCapture;
 			sourceTree = "<group>";
 		};
+		D43546442D8C2AFD00E9C810 /* ViewRedaction */ = {
+			isa = PBXGroup;
+			children = (
+				D43546492D8C32A400E9C810 /* ViewHierarchyNode.swift */,
+				D43546472D8C2B2300E9C810 /* RedactRegion.swift */,
+				D43546452D8C2B0A00E9C810 /* RedactRegionType.swift */,
+				D8AFC0562BDA895400118BE1 /* UIRedactBuilder.swift */,
+			);
+			path = ViewRedaction;
+			sourceTree = "<group>";
+		};
 		D468C0602D36699700964230 /* IO */ = {
 			isa = PBXGroup;
 			children = (
@@ -3913,7 +3930,6 @@
 				D8CAC0722BA4473000E38F34 /* SentryViewPhotographer.swift */,
 				D4E829D52D75E39900D375AD /* SentryViewRenderer.swift */,
 				D8AFC0192BD7A20B00118BE1 /* SentryViewScreenshotProvider.swift */,
-				D8AFC0562BDA895400118BE1 /* UIRedactBuilder.swift */,
 			);
 			path = ViewCapture;
 			sourceTree = "<group>";
@@ -4071,6 +4087,7 @@
 		D856272A2A374A6800FB8062 /* Tools */ = {
 			isa = PBXGroup;
 			children = (
+				D43546442D8C2AFD00E9C810 /* ViewRedaction */,
 				D4E829DD2D75FCA200D375AD /* ViewCapture */,
 				D856272B2A374A8600FB8062 /* UrlSanitized.swift */,
 				D8292D7A2A38AF04009872F7 /* HTTPHeaderSanitizer.swift */,
@@ -4996,6 +5013,7 @@
 				7B63459F280EBA7200CFA05A /* SentryUIEventTracker.m in Sources */,
 				7BF9EF782722B35D00B5BBEF /* SentrySubClassFinder.m in Sources */,
 				D80CD8D32B751447002F710B /* SentryMXCallStackTree.swift in Sources */,
+				D435464A2D8C32A500E9C810 /* ViewHierarchyNode.swift in Sources */,
 				84CFA4CD2C9E0CA3008DA5F4 /* SentryUserFeedbackIntegration.m in Sources */,
 				7BCFA71627D0BB50008C662C /* SentryANRTrackerV1.m in Sources */,
 				8459FCC02BD73EB20038E9C9 /* SentryProfilerSerialization.mm in Sources */,
@@ -5066,6 +5084,7 @@
 				63FE712120DA4C1000CDBAE8 /* SentryCrashSymbolicator.c in Sources */,
 				627C77892D50B6840055E966 /* SentryBreadcrumbCodable.swift in Sources */,
 				63FE70D720DA4C1000CDBAE8 /* SentryCrashMonitor_MachException.c in Sources */,
+				D43546462D8C2B0B00E9C810 /* RedactRegionType.swift in Sources */,
 				7B96572226830D2400C66E25 /* SentryScopeSyncC.c in Sources */,
 				0A9BF4E228A114940068D266 /* SentryViewHierarchyIntegration.m in Sources */,
 				0ADC33EC28D9BB780078D980 /* SentryUIDeviceWrapper.m in Sources */,
@@ -5172,6 +5191,7 @@
 				7BD729982463E93500EA3610 /* SentryDateUtil.m in Sources */,
 				639FCF9D1EBC7F9500778193 /* SentryThread.m in Sources */,
 				D88B30A92D48D8C3008DE513 /* SentryMaskingPreviewView.swift in Sources */,
+				D43546482D8C2B2300E9C810 /* RedactRegion.swift in Sources */,
 				849B8F992C6E906900148E1F /* SentryUserFeedbackFormConfiguration.swift in Sources */,
 				8E8C57A225EEFC07001CEEFA /* SentrySampling.m in Sources */,
 				8454CF8D293EAF9A006AC140 /* SentryMetricProfiler.mm in Sources */,

Sources/Sentry/SentrySessionReplayIntegration.m

@@ -192,6 +192,7 @@
         [[SentryOnDemandReplay alloc] initWithContentFrom:lastReplayURL.path];
     resumeReplayMaker.bitRate = _replayOptions.replayBitRate;
     resumeReplayMaker.videoScale = _replayOptions.sizeScale;
+    resumeReplayMaker.frameRate = _replayOptions.frameRate;
 
     NSDate *beginning = hasCrashInfo
         ? [NSDate dateWithTimeIntervalSinceReferenceDate:crashInfo.lastSegmentEnd]
@@ -319,9 +320,15 @@
     SentryOnDemandReplay *replayMaker = [[SentryOnDemandReplay alloc] initWithOutputPath:docs.path];
     replayMaker.bitRate = replayOptions.replayBitRate;
     replayMaker.videoScale = replayOptions.sizeScale;
-    replayMaker.cacheMaxSize
-        = (NSInteger)(shouldReplayFullSession ? replayOptions.sessionSegmentDuration + 1
-                                              : replayOptions.errorReplayDuration + 1);
+    replayMaker.frameRate = replayOptions.frameRate;
+    replayMaker.onNewFrame = replayOptions.onNewFrame;
+
+    // The cache should be at least the amount of frames fitting into he session segment duration
+    // plus one frame to ensure that the last frame is not dropped.
+    NSInteger sessionSegmentDuration
+        = (NSInteger)(shouldReplayFullSession ? replayOptions.sessionSegmentDuration
+                                              : replayOptions.errorReplayDuration);
+    replayMaker.cacheMaxSize = (sessionSegmentDuration * replayOptions.frameRate) + 1;
 
     dispatch_queue_attr_t attributes = dispatch_queue_attr_make_with_qos_class(
         DISPATCH_QUEUE_SERIAL, DISPATCH_QUEUE_PRIORITY_LOW, 0);

Sources/Sentry/include/SentryDisplayLinkWrapper.m

@@ -21,6 +21,7 @@
 - (void)linkWithTarget:(id)target selector:(SEL)sel
 {
     displayLink = [CADisplayLink displayLinkWithTarget:target selector:sel];
+    displayLink.preferredFramesPerSecond = 10;
     [displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes];
 }
 

Sources/Swift/Integrations/SessionReplay/Preview/SentryMaskingPreviewView.swift

@@ -62,7 +62,7 @@
     private func update() {
         guard let superview = self.superview, idle else { return }
         idle = false
-        self.photographer.image(view: superview) { image in
+        self.photographer.image(view: superview) { _, _, _, image in
             DispatchQueue.main.async {
                 self.imageView.image = image
                 self.idle = true