Back up app APKs in new v2 format

We still support restoring in v1 format for some time.

app/src/androidTest/java/com/stevesoltys/seedvault/backend/saf/SafBackendTest.kt

@@ -13,7 +13,6 @@ import kotlinx.coroutines.runBlocking
 import org.calyxos.seedvault.core.backends.Backend
 import org.calyxos.seedvault.core.backends.BackendTest
 import org.calyxos.seedvault.core.backends.saf.SafBackend
-import org.calyxos.seedvault.core.backends.saf.SafProperties
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.koin.core.component.KoinComponent
@@ -25,14 +24,7 @@ class SafBackendTest : BackendTest(), KoinComponent {
 
     private val context = InstrumentationRegistry.getInstrumentation().targetContext
     private val settingsManager by inject<SettingsManager>()
-    private val safStorage = settingsManager.getSafProperties() ?: error("No SAF storage")
-    private val safProperties = SafProperties(
-        config = safStorage.config,
-        name = safStorage.name,
-        isUsb = safStorage.isUsb,
-        requiresNetwork = safStorage.requiresNetwork,
-        rootId = safStorage.rootId,
-    )
+    private val safProperties = settingsManager.getSafProperties() ?: error("No SAF storage")
     override val backend: Backend = SafBackend(context, safProperties, ".SeedvaultTest")
 
     @Test

app/src/main/java/com/stevesoltys/seedvault/header/Header.kt

@@ -10,7 +10,7 @@ import com.stevesoltys.seedvault.crypto.TYPE_BACKUP_FULL
 import com.stevesoltys.seedvault.crypto.TYPE_BACKUP_KV
 import java.nio.ByteBuffer
 
-internal const val VERSION: Byte = 1
+internal const val VERSION: Byte = 2
 internal const val MAX_PACKAGE_LENGTH_SIZE = 255
 internal const val MAX_KEY_LENGTH_SIZE = MAX_PACKAGE_LENGTH_SIZE
 internal const val MAX_VERSION_HEADER_SIZE =

app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt

@@ -8,8 +8,12 @@ package com.stevesoltys.seedvault.metadata
 import android.content.pm.ApplicationInfo.FLAG_STOPPED
 import android.os.Build
 import com.stevesoltys.seedvault.crypto.TYPE_METADATA
+import com.stevesoltys.seedvault.encodeBase64
 import com.stevesoltys.seedvault.header.VERSION
 import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR
+import com.stevesoltys.seedvault.proto.Snapshot
+import com.stevesoltys.seedvault.transport.backup.hexFromProto
+import com.stevesoltys.seedvault.worker.BASE_SPLIT
 import org.calyxos.backup.storage.crypto.StreamCrypto.toByteArray
 import java.nio.ByteBuffer
 
@@ -91,19 +95,59 @@ data class PackageMetadata(
     internal val version: Long? = null,
     internal val installer: String? = null,
     internal val splits: List<ApkSplit>? = null,
+    internal val chunkIds: List<String>? = null, // used for v2
     internal val sha256: String? = null,
     internal val signatures: List<String>? = null,
 ) {
+
+    companion object {
+        fun fromSnapshot(app: Snapshot.App) = PackageMetadata(
+            time = app.time,
+            state = if (app.state.isBlank()) UNKNOWN_ERROR else PackageState.valueOf(app.state),
+            backupType = when (app.type) {
+                Snapshot.BackupType.FULL -> BackupType.FULL
+                Snapshot.BackupType.KV -> BackupType.KV
+                else -> null
+            },
+            name = app.name,
+            system = app.system,
+            isLaunchableSystemApp = app.launchableSystemApp,
+            version = app.apk.versionCode,
+            installer = app.apk.installer,
+            splits = app.apk.splitsList.filter { it.name != BASE_SPLIT }.map {
+                ApkSplit(
+                    name = it.name,
+                    size = null,
+                    sha256 = "",
+                    chunkIds = if (it.chunkIdsCount == 0) null else it.chunkIdsList.hexFromProto()
+                )
+            }.takeIf { it.isNotEmpty() }, // expected null if there are no splits
+            chunkIds = run {
+                val baseChunk = app.apk.splitsList.find { it.name == BASE_SPLIT }
+                if (baseChunk == null || baseChunk.chunkIdsCount == 0) {
+                    null
+                } else {
+                    baseChunk.chunkIdsList.hexFromProto()
+                }
+            },
+            sha256 = null,
+            signatures = app.apk.signaturesList.map { it.toByteArray().encodeBase64() },
+        )
+    }
+
     val isInternalSystem: Boolean = system && !isLaunchableSystemApp
     fun hasApk(): Boolean {
-        return version != null && sha256 != null && signatures != null
+        return version != null &&
+            (sha256 != null || chunkIds?.isNotEmpty() == true) && // v2 doesn't use sha256 here
+            signatures != null
     }
 }
 
 data class ApkSplit(
     val name: String,
     val size: Long?,
     val sha256: String,
+    val chunkIds: List<String>? = null, // used for v2
     // There's also a revisionCode, but it doesn't seem to be used just yet
 )
 

app/src/main/java/com/stevesoltys/seedvault/restore/install/ApkRestore.kt

@@ -11,32 +11,39 @@ import android.content.Intent
 import android.content.pm.PackageManager
 import android.content.pm.PackageManager.GET_SIGNATURES
 import android.content.pm.PackageManager.GET_SIGNING_CERTIFICATES
+import android.content.pm.SigningInfo
 import android.util.Log
 import com.stevesoltys.seedvault.BackupStateManager
 import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.backend.LegacyStoragePlugin
 import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.encodeBase64
 import com.stevesoltys.seedvault.metadata.ApkSplit
 import com.stevesoltys.seedvault.metadata.PackageMetadata
-import com.stevesoltys.seedvault.backend.BackendManager
-import com.stevesoltys.seedvault.backend.LegacyStoragePlugin
 import com.stevesoltys.seedvault.restore.RestorableBackup
 import com.stevesoltys.seedvault.restore.RestoreService
 import com.stevesoltys.seedvault.restore.install.ApkInstallState.FAILED
 import com.stevesoltys.seedvault.restore.install.ApkInstallState.FAILED_SYSTEM_APP
 import com.stevesoltys.seedvault.restore.install.ApkInstallState.IN_PROGRESS
 import com.stevesoltys.seedvault.restore.install.ApkInstallState.QUEUED
 import com.stevesoltys.seedvault.restore.install.ApkInstallState.SUCCEEDED
+import com.stevesoltys.seedvault.transport.backup.hexFromProto
 import com.stevesoltys.seedvault.transport.backup.isSystemApp
-import com.stevesoltys.seedvault.worker.copyStreamsAndGetHash
-import com.stevesoltys.seedvault.worker.getSignatures
+import com.stevesoltys.seedvault.transport.restore.Loader
+import com.stevesoltys.seedvault.worker.hashSignature
 import kotlinx.coroutines.TimeoutCancellationException
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.coroutines.flow.update
+import org.calyxos.seedvault.core.backends.AppBackupFileType
 import org.calyxos.seedvault.core.backends.Backend
 import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
 import java.io.File
 import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
+import java.security.MessageDigest
 import java.util.Locale
 
 private val TAG = ApkRestore::class.java.simpleName
@@ -46,6 +53,7 @@ internal class ApkRestore(
     private val backupManager: IBackupManager,
     private val backupStateManager: BackupStateManager,
     private val backendManager: BackendManager,
+    private val loader: Loader,
     @Suppress("Deprecation")
     private val legacyStoragePlugin: LegacyStoragePlugin,
     private val crypto: Crypto,
@@ -130,6 +138,7 @@ internal class ApkRestore(
                 Log.e(TAG, "Timeout while re-installing APK for $packageName.", e)
                 mInstallResult.update { it.fail(packageName) }
             } catch (e: Exception) {
+                if (e::class.simpleName == "MockKException") throw e
                 Log.e(TAG, "Unexpected exception while re-installing APK for $packageName.", e)
                 mInstallResult.update { it.fail(packageName) }
             }
@@ -168,10 +177,10 @@ internal class ApkRestore(
         }
 
         // cache the APK and get its hash
-        val (cachedApk, sha256) = cacheApk(backup.version, backup.token, backup.salt, packageName)
+        val (cachedApk, sha256) = cacheApk(backup, packageName, metadata.chunkIds)
 
-        // check APK's SHA-256 hash
-        if (metadata.sha256 != sha256) throw SecurityException(
+        // check APK's SHA-256 hash for backup versions before 2
+        if (backup.version < 2 && metadata.sha256 != sha256) throw SecurityException(
             "Package $packageName has sha256 '$sha256', but '${metadata.sha256}' expected."
         )
 
@@ -262,10 +271,9 @@ internal class ApkRestore(
         }
         splits.forEach { apkSplit -> // cache and check all splits
             val suffix = if (backup.version == 0.toByte()) "_${apkSplit.sha256}" else apkSplit.name
-            val salt = backup.salt
-            val (file, sha256) = cacheApk(backup.version, backup.token, salt, packageName, suffix)
-            // check APK split's SHA-256 hash
-            if (apkSplit.sha256 != sha256) throw SecurityException(
+            val (file, sha256) = cacheApk(backup, packageName, apkSplit.chunkIds, suffix)
+            // check APK split's SHA-256 hash for backup versions before 2
+            if (backup.version < 2 && apkSplit.sha256 != sha256) throw SecurityException(
                 "$packageName:${apkSplit.name} has sha256 '$sha256'," +
                     " but '${apkSplit.sha256}' expected."
             )
@@ -282,20 +290,32 @@ internal class ApkRestore(
      */
     @Throws(IOException::class)
     private suspend fun cacheApk(
-        version: Byte,
-        token: Long,
-        salt: String,
+        backup: RestorableBackup,
         packageName: String,
+        chunkIds: List<String>?,
         suffix: String = "",
     ): Pair<File, String> {
         // create a cache file to write the APK into
         val cachedApk = File.createTempFile(packageName + suffix, ".apk", context.cacheDir)
         // copy APK to cache file and calculate SHA-256 hash while we are at it
-        val inputStream = if (version == 0.toByte()) {
-            legacyStoragePlugin.getApkInputStream(token, packageName, suffix)
-        } else {
-            val name = crypto.getNameForApk(salt, packageName, suffix)
-            backend.load(LegacyAppBackupFile.Blob(token, name))
+        val inputStream = when (backup.version) {
+            0.toByte() -> {
+                legacyStoragePlugin.getApkInputStream(backup.token, packageName, suffix)
+            }
+            1.toByte() -> {
+                val name = crypto.getNameForApk(backup.salt, packageName, suffix)
+                backend.load(LegacyAppBackupFile.Blob(backup.token, name))
+            }
+            else -> {
+                val repoId = backup.repoId ?: error("No repoId for v2 backup")
+                val snapshot = backup.snapshot ?: error("No snapshot for v2 backup")
+                val handles = chunkIds?.map { chunkId ->
+                    val blobId = snapshot.blobsMap[chunkId]?.id?.hexFromProto()
+                        ?: error("Blob for $chunkId missing from snapshot ${snapshot.token}")
+                    AppBackupFileType.Blob(repoId, blobId)
+                } ?: error("No chunkIds for $packageName-$suffix")
+                loader.loadFiles(handles)
+            }
         }
         val sha256 = copyStreamsAndGetHash(inputStream, cachedApk.outputStream())
         return Pair(cachedApk, sha256)
@@ -343,3 +363,45 @@ internal class ApkRestore(
         }
     }
 }
+
+/**
+ * Copy the APK from the given [InputStream] to the given [OutputStream]
+ * and calculate the SHA-256 hash while at it.
+ *
+ * Both streams will be closed when the method returns.
+ *
+ * @return the APK's SHA-256 hash in Base64 format.
+ */
+@Throws(IOException::class)
+fun copyStreamsAndGetHash(inputStream: InputStream, outputStream: OutputStream): String {
+    val messageDigest = MessageDigest.getInstance("SHA-256")
+    outputStream.use { oStream ->
+        inputStream.use { inputStream ->
+            val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
+            var bytes = inputStream.read(buffer)
+            while (bytes >= 0) {
+                oStream.write(buffer, 0, bytes)
+                messageDigest.update(buffer, 0, bytes)
+                bytes = inputStream.read(buffer)
+            }
+        }
+    }
+    return messageDigest.digest().encodeBase64()
+}
+
+/**
+ * Returns a list of Base64 encoded SHA-256 signature hashes.
+ */
+fun SigningInfo?.getSignatures(): List<String> {
+    return if (this == null) {
+        emptyList()
+    } else if (hasMultipleSigners()) {
+        apkContentsSigners.map { signature ->
+            hashSignature(signature).encodeBase64()
+        }
+    } else {
+        signingCertificateHistory.map { signature ->
+            hashSignature(signature).encodeBase64()
+        }
+    }
+}

app/src/main/java/com/stevesoltys/seedvault/restore/install/InstallModule.kt

@@ -14,7 +14,7 @@ val installModule = module {
     factory { DeviceInfo(androidContext()) }
     factory { ApkSplitCompatibilityChecker(get()) }
     factory {
-        ApkRestore(androidContext(), get(), get(), get(), get(), get(), get(), get()) {
+        ApkRestore(androidContext(), get(), get(), get(), get(), get(), get(), get(), get()) {
             androidContext().getSystemService(UserManager::class.java)!!.isAllowedToInstallApks()
         }
     }

app/src/main/java/com/stevesoltys/seedvault/ui/notification/NotificationBackupObserver.kt

@@ -19,8 +19,10 @@ import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER
 import com.stevesoltys.seedvault.R
 import com.stevesoltys.seedvault.metadata.MetadataManager
 import com.stevesoltys.seedvault.settings.SettingsManager
+import com.stevesoltys.seedvault.transport.backup.AppBackupManager
 import com.stevesoltys.seedvault.transport.backup.PackageService
 import com.stevesoltys.seedvault.worker.BackupRequester
+import kotlinx.coroutines.runBlocking
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
 
@@ -36,6 +38,7 @@ internal class NotificationBackupObserver(
     private val metadataManager: MetadataManager by inject()
     private val packageService: PackageService by inject()
     private val settingsManager: SettingsManager by inject()
+    private val appBackupManager: AppBackupManager by inject()
     private var currentPackage: String? = null
     private var numPackages: Int = 0
     private var numPackagesToReport: Int = 0
@@ -141,6 +144,11 @@ internal class NotificationBackupObserver(
                 Log.e(TAG, "Error getting number of all user packages: ", e)
                 requestedPackages
             }
+            // TODO handle exceptions
+            runBlocking {
+                // TODO check if UI thread
+                appBackupManager.afterBackupFinished()
+            }
             nm.onBackupFinished(success, numPackagesToReport, total, size)
         }
     }