feat: Support orientation change for session replay

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- Support orientation change for session replay (#4194)
 - Replay for crashes (#4171)
 - Redact web view from replay (#4203)
 - Add beforeCaptureViewHierarchy callback (#4210)
@@ -18,6 +19,10 @@
 ### Fixes
 
 - Session replay crash when writing the replay (#4186)
+
+### Features
+
+- Replay for crashes (#4171)
 - Collect only unique UIWindow references (#4159)
 
 ### Deprecated

Sources/Sentry/SentrySessionReplayIntegration.m

@@ -4,6 +4,7 @@
 
 #    import "SentryClient+Private.h"
 #    import "SentryDependencyContainer.h"
+#    import "SentryDispatchQueueWrapper.h"
 #    import "SentryDisplayLinkWrapper.h"
 #    import "SentryEvent+Private.h"
 #    import "SentryFileManager.h"
@@ -21,7 +22,6 @@
 #    import "SentrySwizzle.h"
 #    import "SentryUIApplication.h"
 #    import <UIKit/UIKit.h>
-
 NS_ASSUME_NONNULL_BEGIN
 
 static NSString *SENTRY_REPLAY_FOLDER = @"replay";
@@ -42,6 +42,7 @@ @implementation SentrySessionReplayIntegration {
     BOOL _startedAsFullSession;
     SentryReplayOptions *_replayOptions;
     SentryNSNotificationCenterWrapper *_notificationCenter;
+    SentryDispatchQueueWrapper *_dispatchQueue;
     SentryOnDemandReplay *_resumeReplayMaker;
 }
 
@@ -114,39 +115,41 @@ - (void)resumePreviousSessionReplay:(SentryEvent *)event
         }
     }
 
-    _resumeReplayMaker = [[SentryOnDemandReplay alloc] initWithContentFrom:lastReplayURL.path];
-    _resumeReplayMaker.bitRate = _replayOptions.replayBitRate;
-    _resumeReplayMaker.videoScale = _replayOptions.sizeScale;
+    SentryOnDemandReplay *resumeReplayMaker =
+        [[SentryOnDemandReplay alloc] initWithContentFrom:lastReplayURL.path];
+    resumeReplayMaker.bitRate = _replayOptions.replayBitRate;
+    resumeReplayMaker.videoScale = _replayOptions.sizeScale;
 
     NSDate *beginning = hasCrashInfo
         ? [NSDate dateWithTimeIntervalSinceReferenceDate:crashInfo.lastSegmentEnd]
-        : [_resumeReplayMaker oldestFrameDate];
+        : [resumeReplayMaker oldestFrameDate];
 
     if (beginning == nil) {
         return; // no frames to send
     }
 
-    NSError *error;
-    if (![_resumeReplayMaker
+    _dispatchQueue = [[SentryDispatchQueueWrapper alloc] init];
+
+    [_dispatchQueue dispatchAsyncWithBlock:^{
+        SentryReplayType _type = type;
+        int _segmentId = segmentId;
+
+        NSError *error;
+        NSArray<SentryVideoInfo *> *videos = [resumeReplayMaker
             createVideoWithBeginning:beginning
                                  end:[beginning dateByAddingTimeInterval:duration]
-                       outputFileURL:[lastReplayURL URLByAppendingPathComponent:@"lastVideo.mp4"]
-                               error:&error
-                          completion:^(SentryVideoInfo *video, NSError *renderError) {
-                              if (renderError != nil) {
-                                  SENTRY_LOG_ERROR(
-                                      @"Could not create replay video: %@", renderError);
-                              } else {
-                                  [self captureVideo:video
-                                            replayId:replayId
-                                           segmentId:segmentId
-                                                type:type];
-                              }
-                              self->_resumeReplayMaker = nil;
-                          }]) {
-        SENTRY_LOG_ERROR(@"Could not create replay video: %@", error);
-        return;
-    }
+                               error:&error];
+        if (videos == nil) {
+            SENTRY_LOG_ERROR(@"Could not create replay video: %@", error);
+            return;
+        }
+
+        for (SentryVideoInfo *video in videos) {
+            [self captureVideo:video replayId:replayId segmentId:_segmentId++ type:_type];
+            // type buffer is only for the first segment
+            _type = SentryReplayTypeSession;
+        }
+    }];
 
     NSMutableDictionary *eventContext = event.context.mutableCopy;
     eventContext[@"replay"] =
@@ -174,13 +177,15 @@ - (void)captureVideo:(SentryVideoInfo *)video
 
     NSError *error = nil;
     if (![[NSFileManager defaultManager] removeItemAtURL:video.path error:&error]) {
-        NSLog(@"[SentrySessionReplay:%d] Could not delete replay segment from disk: %@", __LINE__,
-            error.localizedDescription);
+        SENTRY_LOG_DEBUG(
+            @"Could not delete replay segment from disk: %@", error.localizedDescription);
     }
 }
 
 - (void)startSession
 {
+    [self.sessionReplay stop];
+
     _startedAsFullSession = [self shouldReplayFullSession:_replayOptions.sessionSampleRate];
 
     if (!_startedAsFullSession && _replayOptions.errorSampleRate == 0) {
@@ -247,6 +252,7 @@ - (void)startWithOptions:(SentryReplayOptions *)replayOptions
                  touchTracker:_touchTracker
                  dateProvider:SentryDependencyContainer.sharedInstance.dateProvider
                      delegate:self
+                dispatchQueue:[[SentryDispatchQueueWrapper alloc] init]
            displayLinkWrapper:[[SentryDisplayLinkWrapper alloc] init]];
 
     [self.sessionReplay
@@ -320,9 +326,6 @@ - (void)sentrySessionEnded:(SentrySession *)session
 
 - (void)sentrySessionStarted:(SentrySession *)session
 {
-    if (_sessionReplay) {
-        return;
-    }
     [self startSession];
 }
 

Sources/Swift/Integrations/SessionReplay/SentryOnDemandReplay.swift

@@ -13,23 +13,16 @@ struct SentryReplayFrame {
     let screenName: String?
 }
 
-private struct VideoFrames {
-    let framesPaths: [String]
-    let screens: [String]
-    let start: Date
-    let end: Date
-}
-
 enum SentryOnDemandReplayError: Error {
     case cantReadVideoSize
-    case assetWriterNotReady
+    case cantCreatePixelBuffer
+    case errorRenderingVideo
 }
 
 @objcMembers
 class SentryOnDemandReplay: NSObject, SentryReplayVideoMaker {
 
     private let _outputPath: String
-    private var _currentPixelBuffer: SentryPixelBuffer?
     private var _totalFrames = 0
     private let dateProvider: SentryCurrentDateProvider
     private let workingQueue: SentryDispatchQueueWrapper
@@ -42,16 +35,10 @@ class SentryOnDemandReplay: NSObject, SentryReplayVideoMaker {
         set { _frames = newValue }
     }
     #endif // TEST || TESTCI || DEBUG
-
-    var videoWidth = 200
-    var videoHeight = 434
     var videoScale: Float = 1
     var bitRate = 20_000
     var frameRate = 1
     var cacheMaxSize = UInt.max
-
-    private var actualWidth: Int { Int(Float(videoWidth) * videoScale) }
-    private var actualHeight: Int { Int(Float(videoHeight) * videoScale) }
 
     init(outputPath: String, workingQueue: SentryDispatchQueueWrapper, dateProvider: SentryCurrentDateProvider) {
         self._outputPath = outputPath
@@ -85,10 +72,6 @@ class SentryOnDemandReplay: NSObject, SentryReplayVideoMaker {
         self.init(withContentFrom: outputPath,
                   workingQueue: SentryDispatchQueueWrapper(name: "io.sentry.onDemandReplay", attributes: nil),
                   dateProvider: SentryCurrentDateProvider())
-
-        guard let last = _frames.last, let image = UIImage(contentsOfFile: last.imagePath) else { return }
-        videoWidth = Int(image.size.width)
-        videoHeight = Int(image.size.height)
     }
 
     func addFrameAsync(image: UIImage, forScreen: String?) {
@@ -140,92 +123,130 @@ class SentryOnDemandReplay: NSObject, SentryReplayVideoMaker {
         return _frames.first?.time
     }
 
-    func createVideoWith(beginning: Date, end: Date, outputFileURL: URL, completion: @escaping (SentryVideoInfo?, Error?) -> Void) throws {
-        var frameCount = 0
+    func createVideoWith(beginning: Date, end: Date) throws -> [SentryVideoInfo] {
         let videoFrames = filterFrames(beginning: beginning, end: end)
-        if videoFrames.framesPaths.isEmpty { return }
+        var frameCount = 0
+
+        var videos = [SentryVideoInfo]()
+
+        while frameCount < videoFrames.count {
+            let outputFileURL = URL(fileURLWithPath: _outputPath.appending("/\(videoFrames[frameCount].time.timeIntervalSinceReferenceDate).mp4"))
+            if let info = try renderVideo(with: videoFrames, from: &frameCount, at: outputFileURL) {
+                videos.append(info)
+            } else {
+                frameCount++
+            }  
+        }
+        return videos
+    }
+
+    private func renderVideo(with videoFrames: [SentryReplayFrame], from: inout Int, at outputFileURL: URL) throws -> SentryVideoInfo? {
+        guard from < videoFrames.count, let image = UIImage(contentsOfFile: videoFrames[from].imagePath) else { return nil }
+        let videoWidth = image.size.width * CGFloat(videoScale)
+        let videoHeight = image.size.height * CGFloat(videoScale)
 
         let videoWriter = try AVAssetWriter(url: outputFileURL, fileType: .mp4)
-        let videoWriterInput = AVAssetWriterInput(mediaType: .video, outputSettings: createVideoSettings())
+        let videoWriterInput = AVAssetWriterInput(mediaType: .video, outputSettings: createVideoSettings(width: videoWidth, height: videoHeight))
 
-        _currentPixelBuffer = SentryPixelBuffer(size: CGSize(width: actualWidth, height: actualHeight), videoWriterInput: videoWriterInput)
-        if _currentPixelBuffer == nil { return }
+        guard let currentPixelBuffer = SentryPixelBuffer(size: CGSize(width: videoWidth, height: videoHeight), videoWriterInput: videoWriterInput)
+        else { throw SentryOnDemandReplayError.cantCreatePixelBuffer }
 
         videoWriter.add(videoWriterInput)
         videoWriter.startWriting()
         videoWriter.startSession(atSourceTime: .zero)
 
+        var lastImageSize: CGSize = image.size
+        var usedFrames = [SentryReplayFrame]()
+        let group = DispatchGroup()
+
+        var result: Result<SentryVideoInfo?, Error>?
+        var frameCount = from
+
+        group.enter()
         videoWriterInput.requestMediaDataWhenReady(on: workingQueue.queue) { [weak self] in
             guard let self = self, videoWriter.status == .writing else {
                 videoWriter.cancelWriting()
-                completion(nil, SentryOnDemandReplayError.assetWriterNotReady)
+                result = .failure(videoWriter.error ?? SentryOnDemandReplayError.errorRenderingVideo )
+                group.leave()
                 return
             }
 
-            if frameCount < videoFrames.framesPaths.count {
-                let imagePath = videoFrames.framesPaths[frameCount]
-                if let image = UIImage(contentsOfFile: imagePath) {
-                    let presentTime = CMTime(seconds: Double(frameCount), preferredTimescale: CMTimeScale(1 / self.frameRate))
-
-                    guard self._currentPixelBuffer?.append(image: image, presentationTime: presentTime) == true 
-                    else {
-                        completion(nil, videoWriter.error)
-                        videoWriterInput.markAsFinished()
-                        return
-                    }
+            if frameCount >= videoFrames.count {
+                result = self.finishVideo(outputFileURL: outputFileURL, usedFrames: usedFrames, videoHeight: Int(videoHeight), videoWidth: Int(videoWidth), videoWriter: videoWriter)
+                group.leave()
+                return
+            }
+
+            let frame = videoFrames[frameCount]
+            if let image = UIImage(contentsOfFile: frame.imagePath) {
+                if lastImageSize != image.size {
+                    result = self.finishVideo(outputFileURL: outputFileURL, usedFrames: usedFrames, videoHeight: Int(videoHeight), videoWidth: Int(videoWidth), videoWriter: videoWriter)
+                    group.leave()
+                    return
                 }
-                frameCount += 1
-            } else {
-                videoWriterInput.markAsFinished()
-                videoWriter.finishWriting {
-                    var videoInfo: SentryVideoInfo?
-                    if videoWriter.status == .completed {
-                        do {
-                            let fileAttributes = try FileManager.default.attributesOfItem(atPath: outputFileURL.path)
-                            guard let fileSize = fileAttributes[FileAttributeKey.size] as? Int else {
-                                completion(nil, SentryOnDemandReplayError.cantReadVideoSize)
-                                return
-                            }
-                            videoInfo = SentryVideoInfo(path: outputFileURL, height: self.actualHeight, width: self.actualWidth, duration: TimeInterval(videoFrames.framesPaths.count / self.frameRate), frameCount: videoFrames.framesPaths.count, frameRate: self.frameRate, start: videoFrames.start, end: videoFrames.end, fileSize: fileSize, screens: videoFrames.screens)
-                        } catch {
-                            completion(nil, error)
-                        }
-                    }
-                    completion(videoInfo, videoWriter.error)
+                lastImageSize = image.size
+
+                let presentTime = CMTime(seconds: Double(frameCount), preferredTimescale: CMTimeScale(1 / self.frameRate))
+
+                if currentPixelBuffer.append(image: image, presentationTime: presentTime) != true {
+                    videoWriter.cancelWriting()
+                    result = .failure(videoWriter.error ?? SentryOnDemandReplayError.errorRenderingVideo )
+                    group.leave()
+                    return
                 }
+                usedFrames.append(frame)
             }
+            frameCount += 1
         }
+        group.wait()
+        from = frameCount
+
+        return try result?.get()
     }
-
-    private func filterFrames(beginning: Date, end: Date) -> VideoFrames {
-        var framesPaths = [String]()
 
-        var screens = [String]()
+    private func finishVideo(outputFileURL: URL, usedFrames: [SentryReplayFrame], videoHeight: Int, videoWidth: Int, videoWriter: AVAssetWriter) -> Result<SentryVideoInfo?, Error> {
+        let group = DispatchGroup()
+        var finishError: Error?
+        var result: SentryVideoInfo?
 
-        var start = dateProvider.date()
-        var actualEnd = start
-        workingQueue.dispatchSync({
-            for frame in self._frames {
-                if frame.time < beginning { continue } else if frame.time > end { break }
-
-                if frame.time < start { start = frame.time }
-
-                if let screenName = frame.screenName {
-                    screens.append(screenName)
+        group.enter()
+        videoWriter.inputs.forEach { $0.markAsFinished() }
+        videoWriter.finishWriting {
+            defer { group.leave() }
+            if videoWriter.status == .completed {
+                do {
+                    let fileAttributes = try FileManager.default.attributesOfItem(atPath: outputFileURL.path)
+                    guard let fileSize = fileAttributes[FileAttributeKey.size] as? Int else {
+                        finishError = SentryOnDemandReplayError.cantReadVideoSize
+                        return
+                    }
+                    guard let start = usedFrames.min(by: { $0.time < $1.time })?.time else { return }
+                    let duration = TimeInterval(usedFrames.count / self.frameRate)
+                    result = SentryVideoInfo(path: outputFileURL, height: Int(videoHeight), width: Int(videoWidth), duration: duration, frameCount: usedFrames.count, frameRate: self.frameRate, start: start, end: start.addingTimeInterval(duration), fileSize: fileSize, screens: usedFrames.compactMap({ $0.screenName }))
+                } catch {
+                    finishError = error
                 }
-
-                actualEnd = frame.time
-                framesPaths.append(frame.imagePath)
             }
+        }
+        group.wait()
+
+        if let finishError = finishError { return .failure(finishError) }
+        return .success(result)
+    }
+
+    private func filterFrames(beginning: Date, end: Date) -> [SentryReplayFrame] {
+        var frames = [SentryReplayFrame]()
+        workingQueue.dispatchSync({
+            frames = self._frames.filter { $0.time >= beginning && $0.time <= end }
         })
-        return VideoFrames(framesPaths: framesPaths, screens: screens, start: start, end: actualEnd + TimeInterval((1 / Double(frameRate))))
+        return frames
     }
 
-    private func createVideoSettings() -> [String: Any] {
+    private func createVideoSettings(width: CGFloat, height: CGFloat) -> [String: Any] {
         return [
             AVVideoCodecKey: AVVideoCodecType.h264,
-            AVVideoWidthKey: actualWidth,
-            AVVideoHeightKey: actualHeight,
+            AVVideoWidthKey: width,
+            AVVideoHeightKey: height,
             AVVideoCompressionPropertiesKey: [
                 AVVideoAverageBitRateKey: bitRate,
                 AVVideoProfileLevelKey: AVVideoProfileLevelH264BaselineAutoLevel

Sources/Swift/Integrations/SessionReplay/SentryReplayVideoMaker.swift

@@ -4,12 +4,9 @@ import UIKit
 
 @objc
 protocol SentryReplayVideoMaker: NSObjectProtocol {
-    var videoWidth: Int { get set }
-    var videoHeight: Int { get set }
-
     func addFrameAsync(image: UIImage, forScreen: String?) 
     func releaseFramesUntil(_ date: Date)
-    func createVideoWith(beginning: Date, end: Date, outputFileURL: URL, completion: @escaping (SentryVideoInfo?, Error?) -> Void) throws
+    func createVideoWith(beginning: Date, end: Date) throws -> [SentryVideoInfo]
 }
 
 extension SentryReplayVideoMaker {