[SR] Add screenshot recorder

buildSrc/src/main/java/Config.kt

@@ -34,7 +34,7 @@ object Config {
 
         val minSdkVersion = 19
         val minSdkVersionOkHttp = 21
-        val minSdkVersionReplay = 26
+        val minSdkVersionReplay = 19
         val minSdkVersionNdk = 19
         val minSdkVersionCompose = 21
         val targetSdkVersion = sdkVersion

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -1,9 +1,11 @@
 package io.sentry.android.replay
 
+import android.annotation.TargetApi
 import android.graphics.Bitmap
 import android.graphics.Canvas
-import android.graphics.Color
+import android.graphics.Matrix
 import android.graphics.Paint
+import android.graphics.Rect
 import android.graphics.RectF
 import android.os.Handler
 import android.os.HandlerThread
@@ -20,14 +22,26 @@ import java.lang.ref.WeakReference
 import java.util.WeakHashMap
 import kotlin.system.measureTimeMillis
 
+// TODO: use ILogger of Sentry and change level
+@TargetApi(26)
 internal class ScreenshotRecorder(
-    val rootView: WeakReference<View>,
     val encoder: SimpleVideoEncoder
 ) : ViewTreeObserver.OnDrawListener {
 
+    private var rootView: WeakReference<View>? = null
     private val thread = HandlerThread("SentryReplay").also { it.start() }
     private val handler = Handler(thread.looper)
     private val bitmapToVH = WeakHashMap<Bitmap, ViewHierarchyNode>()
+    private val maskingPaint = Paint()
+    private val singlePixelBitmap: Bitmap = Bitmap.createBitmap(
+        1,
+        1,
+        Bitmap.Config.ARGB_8888
+    )
+    private val singlePixelBitmapCanvas: Canvas = Canvas(singlePixelBitmap)
+    private val prescaledMatrix = Matrix().apply {
+        preScale(encoder.muxerConfig.scaleFactor, encoder.muxerConfig.scaleFactor)
+    }
 
     companion object {
         const val TAG = "ScreenshotRecorder"
@@ -37,12 +51,12 @@ internal class ScreenshotRecorder(
     override fun onDraw() {
         // TODO: replace with Debouncer from sentry-core
         val now = SystemClock.uptimeMillis()
-        if (lastCapturedAtMs != null && (now - lastCapturedAtMs!!) < 1000L) {
+        if (lastCapturedAtMs != null && (now - lastCapturedAtMs!!) < 500L) {
             return
         }
         lastCapturedAtMs = now
 
-        val root = rootView.get()
+        val root = rootView?.get()
         if (root == null || root.width <= 0 || root.height <= 0 || !root.isShown) {
             return
         }
@@ -53,7 +67,6 @@ internal class ScreenshotRecorder(
             root.height,
             Bitmap.Config.ARGB_8888
         )
-        Log.e("BITMAP CREATED", bitmap.toString())
 
         val time = measureTimeMillis {
             val rootNode = ViewHierarchyNode.fromView(root)
@@ -62,8 +75,7 @@ internal class ScreenshotRecorder(
         }
         Log.e("TIME", time.toString())
 
-//    val latch = CountDownLatch(1)
-
+        // postAtFrontOfQueue to ensure the view hierarchy and bitmap are ase close in-sync as possible
         Handler(Looper.getMainLooper()).postAtFrontOfQueue {
             PixelCopy.request(
                 window,
@@ -78,26 +90,37 @@ internal class ScreenshotRecorder(
                     Log.e("BITMAP CAPTURED", bitmap.toString())
                     val viewHierarchy = bitmapToVH[bitmap]
 
-                    if (viewHierarchy != null) {
-                        val canvas = Canvas(bitmap)
+                    val scaledBitmap = Bitmap.createScaledBitmap(
+                        bitmap,
+                        encoder.muxerConfig.videoWidth,
+                        encoder.muxerConfig.videoHeight,
+                        true
+                    )
+
+                    if (viewHierarchy == null) {
+                        Log.e(TAG, "Failed to determine view hierarchy, not capturing")
+                        return@request
+                    } else {
+                        val canvas = Canvas(scaledBitmap)
+                        canvas.setMatrix(prescaledMatrix)
                         viewHierarchy.traverse {
                             if (it.shouldRedact && (it.width > 0 && it.height > 0)) {
                                 it.visibleRect ?: return@traverse
 
-                                val paint = Paint().apply {
-                                    color = it.dominantColor ?: Color.BLACK
+                                // TODO: check for view type rather than rely on absence of dominantColor here
+                                val color = if (it.dominantColor == null) {
+                                    singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null)
+                                    singlePixelBitmap.getPixel(0, 0)
+                                } else {
+                                    it.dominantColor
                                 }
-                                canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, paint)
+
+                                maskingPaint.setColor(color)
+                                canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint)
                             }
                         }
                     }
 
-                    val scaledBitmap = Bitmap.createScaledBitmap(
-                        bitmap,
-                        encoder.muxerConfig.videoWidth,
-                        encoder.muxerConfig.videoHeight,
-                        true
-                    )
 //        val baos = ByteArrayOutputStream()
 //        scaledBitmap.compress(Bitmap.CompressFormat.JPEG, 75, baos)
 //        val bmp = BitmapFactory.decodeByteArray(baos.toByteArray(), 0, baos.size())
@@ -106,14 +129,30 @@ internal class ScreenshotRecorder(
                     scaledBitmap.recycle()
                     bitmap.recycle()
                     Log.i(TAG, "Captured a screenshot")
-//          latch.countDown()
                 },
                 handler
             )
         }
+    }
+
+    fun bind(root: View) {
+        // first unbind the current root
+        unbind(rootView?.get())
+        rootView?.clear()
+
+        // next bind the new root
+        rootView = WeakReference(root)
+        root.viewTreeObserver?.addOnDrawListener(this)
+    }
+
+    fun unbind(root: View?) {
+        root?.viewTreeObserver?.removeOnDrawListener(this)
+    }
 
-//    val success = latch.await(200, MILLISECONDS)
-//    Log.i(TAG, "Captured a screenshot: $success")
+    fun close() {
+        unbind(rootView?.get())
+        rootView?.clear()
+        thread.quitSafely()
     }
 
     private fun ViewHierarchyNode.traverse(callback: (ViewHierarchyNode) -> Unit) {

sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt

@@ -1,21 +1,21 @@
 package io.sentry.android.replay
 
+import android.annotation.TargetApi
 import android.content.Context
 import android.graphics.Point
 import android.os.Build.VERSION
 import android.os.Build.VERSION_CODES
 import android.view.View
-import android.view.ViewTreeObserver
 import android.view.WindowManager
 import io.sentry.android.replay.video.MuxerConfig
 import io.sentry.android.replay.video.SimpleVideoEncoder
 import java.io.File
 import java.lang.ref.WeakReference
-import java.util.WeakHashMap
 import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.LazyThreadSafetyMode.NONE
 import kotlin.math.roundToInt
 
+@TargetApi(26)
 class WindowRecorder {
 
     private val rootViewsSpy by lazy(NONE) {
@@ -24,28 +24,20 @@ class WindowRecorder {
 
     private var encoder: SimpleVideoEncoder? = null
     private val isRecording = AtomicBoolean(false)
-    private val recorders: WeakHashMap<View, ViewTreeObserver.OnDrawListener> = WeakHashMap()
+    private val rootViews = ArrayList<WeakReference<View>>()
+    private var recorder: ScreenshotRecorder? = null
 
     private val onRootViewsChangedListener = OnRootViewsChangedListener { root, added ->
         if (added) {
-            if (recorders.containsKey(root)) {
-                // TODO: log
-                return@OnRootViewsChangedListener
-            }
-            // stop tracking other windows so they don't interfere in the recording like a 25th frame effect
-            recorders.entries.forEach {
-                it.key.viewTreeObserver.removeOnDrawListener(it.value)
-            }
-
-            val recorder = ScreenshotRecorder(WeakReference(root), encoder!!)
-            recorders[root] = recorder
-            root.viewTreeObserver?.addOnDrawListener(recorder)
+            rootViews.add(WeakReference(root))
+            recorder?.bind(root)
         } else {
-            root.viewTreeObserver?.removeOnDrawListener(recorders[root])
-            recorders.remove(root)
+            recorder?.unbind(root)
+            rootViews.removeAll { it.get() == root }
 
-            recorders.entries.forEach {
-                it.key.viewTreeObserver.addOnDrawListener(it.value)
+            val newRoot = rootViews.lastOrNull()?.get()
+            if (newRoot != null && root != newRoot) {
+                recorder?.bind(newRoot)
             }
         }
     }
@@ -62,13 +54,16 @@ class WindowRecorder {
 //            context.resources.displayMetrics.density).roundToInt()
         // TODO: API level check
         // PixelCopy takes screenshots including system bars, so we have to get the real size here
+        val height: Int
         val aspectRatio = if (VERSION.SDK_INT >= VERSION_CODES.R) {
-            wm.currentWindowMetrics.bounds.bottom.toFloat() / wm.currentWindowMetrics.bounds.right.toFloat()
+            height = wm.currentWindowMetrics.bounds.bottom
+            height.toFloat() / wm.currentWindowMetrics.bounds.right.toFloat()
         } else {
             val screenResolution = Point()
             @Suppress("DEPRECATION")
             wm.defaultDisplay.getRealSize(screenResolution)
-            screenResolution.y.toFloat() / screenResolution.x.toFloat()
+            height = screenResolution.y
+            height.toFloat() / screenResolution.x.toFloat()
         }
 
         val videoFile = File(context.cacheDir, "sentry-sr.mp4")
@@ -77,21 +72,23 @@ class WindowRecorder {
                 videoFile,
                 videoWidth = (720 / aspectRatio).roundToInt(),
                 videoHeight = 720,
-                frameRate = 1f,
+                scaleFactor = 720f / height,
+                frameRate = 2f,
                 bitrate = 500 * 1000
             )
-        )
-        encoder?.start()
+        ).also { it.start() }
+        recorder = ScreenshotRecorder(encoder!!)
         rootViewsSpy.listeners += onRootViewsChangedListener
     }
 
     fun stopRecording() {
         rootViewsSpy.listeners -= onRootViewsChangedListener
-        recorders.entries.forEach {
-            it.key.viewTreeObserver.removeOnDrawListener(it.value)
-        }
-        recorders.clear()
+        rootViews.forEach { recorder?.unbind(it.get()) }
+        recorder?.close()
+        rootViews.clear()
+        recorder = null
         encoder?.startRelease()
         encoder = null
+        isRecording.set(false)
     }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt

@@ -29,6 +29,7 @@
  */
 package io.sentry.android.replay.video
 
+import android.annotation.TargetApi
 import android.graphics.Bitmap
 import android.media.MediaCodec
 import android.media.MediaCodecInfo
@@ -38,6 +39,7 @@ import android.os.Looper
 import android.view.Surface
 import java.io.File
 
+@TargetApi(26)
 internal class SimpleVideoEncoder(
     val muxerConfig: MuxerConfig
 ) {
@@ -156,10 +158,12 @@ internal class SimpleVideoEncoder(
     }
 }
 
+@TargetApi(24)
 internal data class MuxerConfig(
     val file: File,
     val videoWidth: Int,
     val videoHeight: Int,
+    val scaleFactor: Float,
     val mimeType: String = MediaFormat.MIMETYPE_VIDEO_AVC,
     val frameRate: Float,
     val bitrate: Int,

sentry-android-replay/src/main/java/io/sentry/android/replay/viewhierarchy/ViewHierarchyNode.kt

@@ -1,7 +1,6 @@
 package io.sentry.android.replay.viewhierarchy
 
-import android.graphics.Bitmap
-import android.graphics.Canvas
+import android.annotation.TargetApi
 import android.graphics.Color
 import android.graphics.Rect
 import android.graphics.drawable.BitmapDrawable
@@ -18,6 +17,7 @@ import android.widget.ImageView
 import android.widget.TextView
 
 // TODO: merge with ViewHierarchyNode from sentry-core maybe?
+@TargetApi(26)
 data class ViewHierarchyNode(
     val x: Float,
     val y: Float,
@@ -47,11 +47,11 @@ data class ViewHierarchyNode(
             return actualPosition.intersects(screen.left, screen.top, screen.right, screen.bottom)
         }
 
-        fun adjustAlpha(color: Int): Int {
+        fun Int.toOpaque(): Int {
             val alpha = 255
-            val red = Color.red(color)
-            val green = Color.green(color)
-            val blue = Color.blue(color)
+            val red = Color.red(this)
+            val green = Color.green(this)
+            val blue = Color.blue(this)
             return Color.argb(alpha, red, green, blue)
         }
 
@@ -77,7 +77,7 @@ data class ViewHierarchyNode(
                         val bounds = Rect()
                         val text = view.text.toString()
                         view.paint.getTextBounds(text, 0, text.length, bounds)
-                        dominantColor = adjustAlpha(view.currentTextColor)
+                        dominantColor = view.currentTextColor.toOpaque()
                         rect = Rect()
                         view.getGlobalVisibleRect(rect)
 
@@ -110,7 +110,6 @@ data class ViewHierarchyNode(
                 is ImageView -> {
                     shouldRedact = isVisible(view) && (view.drawable?.isRedactable() ?: false)
                     if (shouldRedact) {
-                        dominantColor = adjustAlpha((view.drawable?.pickDominantColor() ?: Color.BLACK))
                         rect = Rect()
                         view.getGlobalVisibleRect(rect)
                     }
@@ -136,40 +135,5 @@ data class ViewHierarchyNode(
                 else -> true
             }
         }
-
-        private fun Drawable.pickDominantColor(): Int {
-            // TODO: pick default color based on dark/light default theme
-            return when (this) {
-                is BitmapDrawable -> {
-                    val newBitmap = Bitmap.createScaledBitmap(bitmap, 1, 1, true)
-                    val color = newBitmap.getPixel(0, 0)
-                    newBitmap.recycle()
-                    color
-                }
-
-                else -> {
-                    if (intrinsicHeight > 0 && intrinsicWidth > 0) {
-                        // this is needed to pick a dominant color when there's a drawable from a 3rd-party image-loading lib, e.g. Coil
-                        // we request the bitmap to draw onto the canvas and then downscale it to 1x1 pixels to get the dominant color
-                        // TODO: maybe we should provide an option to disable this and just use black color for rectangles to save cpu time
-                        val bmp =
-                            Bitmap.createBitmap(this.intrinsicWidth, intrinsicHeight, Bitmap.Config.ARGB_8888)
-                        val canvas = Canvas(bmp)
-                        try {
-                            draw(canvas)
-                            val newBitmap = Bitmap.createScaledBitmap(bmp, 1, 1, true)
-                            val color = newBitmap.getPixel(0, 0)
-                            newBitmap.recycle()
-                            bmp.recycle()
-                            color
-                        } catch (e: Throwable) {
-                            Color.BLACK
-                        }
-                    } else {
-                        Color.BLACK
-                    }
-                }
-            }
-        }
     }
 }