Skip to content

Commit

Permalink
WIP: v2 dedup prototype
Browse files Browse the repository at this point in the history
  • Loading branch information
grote committed Sep 6, 2024
1 parent 28753ba commit 691d913
Show file tree
Hide file tree
Showing 38 changed files with 2,293 additions and 605 deletions.
3 changes: 3 additions & 0 deletions Android.bp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ android_app {
"com.google.android.material_material",
"kotlinx-coroutines-android",
"kotlinx-coroutines-core",
// app backup related libs
"seedvault-lib-kotlin-logging-jvm",
"seedvault-lib-chunker"
"seedvault-lib-zstd-jni",
// our own gradle module libs
"seedvault-lib-core",
Expand Down
5 changes: 5 additions & 0 deletions app/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,8 @@ dependencies {

implementation(libs.google.protobuf.javalite)
implementation(libs.google.tink.android)
implementation(libs.kotlin.logging)
implementation(libs.squareup.okio)

/**
* Storage Dependencies
Expand All @@ -175,6 +177,7 @@ dependencies {
implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.jar"))
implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.aar"))

implementation(fileTree("${rootProject.rootDir}/libs").include("seedvault-chunker-0.1.jar"))
implementation(fileTree("${rootProject.rootDir}/libs").include("zstd-jni-1.5.6-5.aar"))
implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar"))

Expand All @@ -188,6 +191,7 @@ dependencies {
// anything less than 'implementation' fails tests run with gradlew
testImplementation(aospLibs)
testImplementation("androidx.test.ext:junit:1.1.5")
testImplementation("org.slf4j:slf4j-simple:2.0.3")
testImplementation("org.robolectric:robolectric:4.12.2")
testImplementation("org.hamcrest:hamcrest:2.2")
testImplementation("org.junit.jupiter:junit-jupiter-api:${libs.versions.junit5.get()}")
Expand All @@ -198,6 +202,7 @@ dependencies {
)
testImplementation("app.cash.turbine:turbine:1.0.0")
testImplementation("org.bitcoinj:bitcoinj-core:0.16.2")
testImplementation("com.github.luben:zstd-jni:1.5.6-5")
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${libs.versions.junit5.get()}")
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import kotlinx.coroutines.runBlocking
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.BackendTest
import org.calyxos.seedvault.core.backends.saf.SafBackend
import org.calyxos.seedvault.core.backends.saf.SafProperties
import org.junit.Test
import org.junit.runner.RunWith
import org.koin.core.component.KoinComponent
Expand All @@ -25,14 +24,7 @@ class SafBackendTest : BackendTest(), KoinComponent {

private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val settingsManager by inject<SettingsManager>()
private val safStorage = settingsManager.getSafProperties() ?: error("No SAF storage")
private val safProperties = SafProperties(
config = safStorage.config,
name = safStorage.name,
isUsb = safStorage.isUsb,
requiresNetwork = safStorage.requiresNetwork,
rootId = safStorage.rootId,
)
private val safProperties = settingsManager.getSafProperties() ?: error("No SAF storage")
override val backend: Backend = SafBackend(context, safProperties, ".SeedvaultTest")

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/*
* SPDX-FileCopyrightText: 2024 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/

package com.stevesoltys.seedvault.worker

import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.MediumTest
import androidx.test.platform.app.InstrumentationRegistry
import com.google.protobuf.ByteString
import com.stevesoltys.seedvault.proto.Snapshot
import com.stevesoltys.seedvault.transport.backup.AppBackupManager
import com.stevesoltys.seedvault.transport.backup.BackupData
import com.stevesoltys.seedvault.transport.backup.BackupReceiver
import com.stevesoltys.seedvault.transport.backup.PackageService
import com.stevesoltys.seedvault.transport.backup.SnapshotCreatorFactory
import com.stevesoltys.seedvault.transport.restore.Loader
import io.mockk.Runs
import io.mockk.coEvery
import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.slot
import junit.framework.TestCase.assertTrue
import kotlinx.coroutines.runBlocking
import org.calyxos.seedvault.core.backends.AppBackupFileType
import org.calyxos.seedvault.core.toHexString
import org.junit.Assert.assertArrayEquals
import org.junit.Test
import org.junit.runner.RunWith
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import kotlin.random.Random

@RunWith(AndroidJUnit4::class)
@MediumTest
class IconManagerTest : KoinComponent {

private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val packageService by inject<PackageService>()
private val backupReceiver = mockk<BackupReceiver>()
private val loader = mockk<Loader>()
private val appBackupManager = mockk<AppBackupManager>()
private val snapshotCreatorFactory by inject<SnapshotCreatorFactory>()
private val snapshotCreator = snapshotCreatorFactory.createSnapshotCreator()

private val iconManager = IconManager(
context = context,
packageService = packageService,
backupReceiver = backupReceiver,
loader = loader,
appBackupManager = appBackupManager,
)

init {
every { appBackupManager.snapshotCreator } returns snapshotCreator
}

@Test
fun `test upload and then download`(): Unit = runBlocking {
// prepare output data
val output = slot<ByteArray>()
val chunkId = Random.nextBytes(32).toHexString()
val chunkList = listOf(chunkId)
val blobId = Random.nextBytes(32).toHexString()
val blob = Snapshot.Blob.newBuilder().setId(ByteString.fromHex(blobId)).build()

// upload icons and capture plaintext bytes
coEvery { backupReceiver.addBytes(capture(output)) } just Runs
coEvery { backupReceiver.finalize() } returns BackupData(chunkList, mapOf(chunkId to blob))
iconManager.uploadIcons()
assertTrue(output.captured.isNotEmpty())

// get snapshot and assert it has icon chunks
val snapshot = snapshotCreator.finalizeSnapshot()
assertTrue(snapshot.iconChunkIdsCount > 0)

// prepare data for downloading icons
val repoId = Random.nextBytes(32).toHexString()
val inputStream = ByteArrayInputStream(output.captured)
coEvery { loader.loadFile(AppBackupFileType.Blob(repoId, blobId)) } returns inputStream

// download icons and ensure we had an icon for at least one app
val iconSet = iconManager.downloadIcons(repoId, snapshot)
assertTrue(iconSet.isNotEmpty())
}

@Test
fun `test upload produces deterministic output`(): Unit = runBlocking {
val output1 = slot<ByteArray>()
val output2 = slot<ByteArray>()

coEvery { backupReceiver.addBytes(capture(output1)) } just Runs
coEvery { backupReceiver.finalize() } returns BackupData(emptyList(), emptyMap())
iconManager.uploadIcons()
assertTrue(output1.captured.isNotEmpty())

coEvery { backupReceiver.addBytes(capture(output2)) } just Runs
coEvery { backupReceiver.finalize() } returns BackupData(emptyList(), emptyMap())
iconManager.uploadIcons()
assertTrue(output2.captured.isNotEmpty())

assertArrayEquals(output1.captured, output2.captured)
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import com.stevesoltys.seedvault.crypto.TYPE_BACKUP_FULL
import com.stevesoltys.seedvault.crypto.TYPE_BACKUP_KV
import java.nio.ByteBuffer

internal const val VERSION: Byte = 1
internal const val VERSION: Byte = 2
internal const val MAX_PACKAGE_LENGTH_SIZE = 255
internal const val MAX_KEY_LENGTH_SIZE = MAX_PACKAGE_LENGTH_SIZE
internal const val MAX_VERSION_HEADER_SIZE =
Expand Down
46 changes: 45 additions & 1 deletion app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ package com.stevesoltys.seedvault.metadata
import android.content.pm.ApplicationInfo.FLAG_STOPPED
import android.os.Build
import com.stevesoltys.seedvault.crypto.TYPE_METADATA
import com.stevesoltys.seedvault.encodeBase64
import com.stevesoltys.seedvault.header.VERSION
import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR
import com.stevesoltys.seedvault.proto.Snapshot
import com.stevesoltys.seedvault.transport.backup.hexFromProto
import com.stevesoltys.seedvault.worker.BASE_SPLIT
import org.calyxos.backup.storage.crypto.StreamCrypto.toByteArray
import java.nio.ByteBuffer

Expand Down Expand Up @@ -91,19 +95,59 @@ data class PackageMetadata(
internal val version: Long? = null,
internal val installer: String? = null,
internal val splits: List<ApkSplit>? = null,
internal val chunkIds: List<String>? = null, // used for v2
internal val sha256: String? = null,
internal val signatures: List<String>? = null,
) {

companion object {
fun fromSnapshot(app: Snapshot.App) = PackageMetadata(
time = app.time,
state = if (app.state.isBlank()) UNKNOWN_ERROR else PackageState.valueOf(app.state),
backupType = when (app.type) {
Snapshot.BackupType.FULL -> BackupType.FULL
Snapshot.BackupType.KV -> BackupType.KV
else -> null
},
name = app.name,
system = app.system,
isLaunchableSystemApp = app.launchableSystemApp,
version = app.apk.versionCode,
installer = app.apk.installer,
splits = app.apk.splitsList.filter { it.name != BASE_SPLIT }.map {
ApkSplit(
name = it.name,
size = null,
sha256 = "",
chunkIds = if (it.chunkIdsCount == 0) null else it.chunkIdsList.hexFromProto()
)
}.takeIf { it.isNotEmpty() }, // expected null if there are no splits
chunkIds = run {
val baseChunk = app.apk.splitsList.find { it.name == BASE_SPLIT }
if (baseChunk == null || baseChunk.chunkIdsCount == 0) {
null
} else {
baseChunk.chunkIdsList.hexFromProto()
}
},
sha256 = null,
signatures = app.apk.signaturesList.map { it.toByteArray().encodeBase64() },
)
}

val isInternalSystem: Boolean = system && !isLaunchableSystemApp
fun hasApk(): Boolean {
return version != null && sha256 != null && signatures != null
return version != null &&
(sha256 != null || chunkIds?.isNotEmpty() == true) && // v2 doesn't use sha256 here
signatures != null
}
}

data class ApkSplit(
val name: String,
val size: Long?,
val sha256: String,
val chunkIds: List<String>? = null, // used for v2
// There's also a revisionCode, but it doesn't seem to be used just yet
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ import androidx.lifecycle.asLiveData
import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER
import com.stevesoltys.seedvault.NO_DATA_END_SENTINEL
import com.stevesoltys.seedvault.R
import com.stevesoltys.seedvault.backend.BackendManager
import com.stevesoltys.seedvault.metadata.PackageMetadata
import com.stevesoltys.seedvault.metadata.PackageMetadataMap
import com.stevesoltys.seedvault.backend.BackendManager
import com.stevesoltys.seedvault.proto.Snapshot
import com.stevesoltys.seedvault.ui.PACKAGE_NAME_SYSTEM
import com.stevesoltys.seedvault.ui.systemData
import com.stevesoltys.seedvault.worker.IconManager
Expand All @@ -24,7 +25,6 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
import java.util.Locale

internal class SelectedAppsState(
Expand All @@ -37,7 +37,7 @@ private val TAG = AppSelectionManager::class.simpleName

internal class AppSelectionManager(
private val context: Context,
private val backendManager: BackendManager,
private val backendManager: BackendManager, // TODO remove
private val iconManager: IconManager,
private val coroutineScope: CoroutineScope,
private val workDispatcher: CoroutineDispatcher = Dispatchers.IO,
Expand Down Expand Up @@ -88,12 +88,11 @@ internal class AppSelectionManager(
SelectedAppsState(apps = items, allSelected = isSetupWizard, iconsLoaded = false)
// download icons
coroutineScope.launch(workDispatcher) {
val backend = backendManager.backend
val token = restorableBackup.token
val packagesWithIcons = try {
backend.load(LegacyAppBackupFile.IconsFile(token)).use {
iconManager.downloadIcons(restorableBackup.version, token, it)
}
// TODO get real repoId
val repoId = "3f1f3d9da0fd5a509196cc96b75c668172592fcb5c20b9159f398da2b6149cc1"
// TODO get real snapshot
iconManager.downloadIcons(repoId, Snapshot.newBuilder().build())
} catch (e: Exception) {
Log.e(TAG, "Error loading icons:", e)
emptySet()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,13 @@ package com.stevesoltys.seedvault.restore

import com.stevesoltys.seedvault.metadata.BackupMetadata
import com.stevesoltys.seedvault.metadata.PackageMetadataMap
import com.stevesoltys.seedvault.proto.Snapshot

data class RestorableBackup(val backupMetadata: BackupMetadata) {
data class RestorableBackup(
val backupMetadata: BackupMetadata,
val repoId: String? = null,
val snapshot: Snapshot? = null,
) {

val name: String
get() = backupMetadata.deviceName
Expand Down
Loading

0 comments on commit 691d913

Please sign in to comment.