Merge branch 'witness-rs' of https://github.com/worldcoin/semaphore-rs …

…into witness-rs

Cargo.toml

@@ -37,6 +37,7 @@ ark-ff = { version = "0.3.0", default-features = false, features = ["parallel",
 ark-groth16 = { git = "https://github.com/arkworks-rs/groth16", rev = "765817f", features = ["parallel"] }
 ark-relations = { version = "0.3.0", default-features = false }
 ark-std = { version = "0.3.0", default-features = false, features = ["parallel"] }
+ark-zkey = { path = "crates/ark-zkey" }
 color-eyre = "0.6"
 criterion = { version = "0.3", optional = true, features = ["async_tokio"] }
 hex = "0.4.0"
@@ -73,6 +74,7 @@ tiny-keccak = "2.0.2"
 tracing-test = "0.2"
 
 [build-dependencies]
+ark-zkey = { path = "crates/ark-zkey" }
 color-eyre = "0.6"
 enumset = "1.0.8"
 reqwest = { version = "0.11", features = ["blocking"] }

build.rs

@@ -6,6 +6,8 @@ use std::{
 
 extern crate reqwest;
 
+use ark_zkey;
+
 const SEMAPHORE_FILES_PATH: &str = "semaphore_files";
 const SEMAPHORE_DOWNLOAD_URL: &str = "https://www.trusted-setup-pse.org/semaphore";
 
@@ -44,6 +46,23 @@ fn semaphore_file_path(file_name: &str, depth: usize) -> PathBuf {
         .join(file_name)
 }
 
+fn create_arkzkey(path: PathBuf) -> Result<PathBuf> {
+    let ark_zkey_path = path.join("-arkzkey");
+
+    let (original_proving_key, original_constraint_matrices) =
+        ark_zkey::read_proving_key_and_matrices_from_zkey(
+            path.to_str().expect("Failed to convert path."),
+        )?;
+
+    ark_zkey::convert_zkey(
+        original_proving_key,
+        original_constraint_matrices,
+        &ark_zkey_path.to_str().unwrap(),
+    )?;
+
+    Ok(ark_zkey_path)
+}
+
 fn build_circuit(depth: usize) -> Result<()> {
     let base_path = Path::new(SEMAPHORE_FILES_PATH);
     if !base_path.exists() {
@@ -63,15 +82,20 @@ fn build_circuit(depth: usize) -> Result<()> {
         let download_url = format!("{SEMAPHORE_DOWNLOAD_URL}/{depth_str}/{filename}.{extension}");
         let path = Path::new(&depth_subfolder).join(format!("{filename}.{extension}"));
         download_and_store_binary(&download_url, &path)?;
+        create_arkzkey(path)?;
     }
 
     // Compute absolute paths
     let zkey_file = absolute(semaphore_file_path("semaphore.zkey", depth))?;
-    let graph_file = absolute(Path::new("graphs")
-    .join(depth.to_string())
-    .join("graph.bin"))?;
+    let arkzkey_file = absolute(semaphore_file_path("semaphore.zkey-arkzkey", depth))?;
+    let graph_file = absolute(
+        Path::new("graphs")
+            .join(depth.to_string())
+            .join("graph.bin"),
+    )?;
 
     assert!(zkey_file.exists());
+    assert!(arkzkey_file.exists());
     assert!(graph_file.exists());
 
     // Export generated paths
@@ -80,6 +104,11 @@ fn build_circuit(depth: usize) -> Result<()> {
         depth,
         zkey_file.display()
     );
+    println!(
+        "cargo:rustc-env=BUILD_RS_ARKZKEY_FILE_{}={}",
+        depth,
+        arkzkey_file.display()
+    );
     println!(
         "cargo:rustc-env=BUILD_RS_GRAPH_FILE_{}={}",
         depth,

crates/ark-zkey/.gitignore

@@ -0,0 +1,14 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb

crates/ark-zkey/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "ark-zkey"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+# XXX: Shouldn't be necessary, but this way we stay consistent with wasmer version and fix
+# error[E0432]: unresolved import `wasmer` error
+# (likely due to other packages)
+[patch.crates-io]
+# NOTE: Forked wasmer to work around memory limits
+# See https://github.com/wasmerio/wasmer/commit/09c7070
+wasmer = { git = "https://github.com/oskarth/wasmer.git", rev = "09c7070" }
+
+[dependencies]
+color-eyre = "0.6"
+memmap2 = "0.9"
+flame = "0.2"
+flamer = "0.5"
+
+ark-serialize = { version = "=0.4.1", features = ["derive"] }
+ark-bn254 = { version = "=0.4.0" }
+ark-groth16 = { version = "=0.4.0" }
+ark-circom = { git = "https://github.com/arkworks-rs/circom-compat.git" }
+ark-relations = { version = "=0.4.0" }
+ark-ff = { version = "=0.4.1" }
+ark-ec = { version = "=0.4.1" }

crates/ark-zkey/README.md

@@ -0,0 +1,79 @@
+# ark-zkey
+
+Library to read `zkey` faster by serializing to `arkworks` friendly format.
+
+See https://github.com/oskarth/mopro/issues/25 for context.
+
+## To generate arkzkey
+
+Hacky, but the way we generate `arkzkey` now is by running the corresponding test.
+
+Note that we also neeed to change the const `ZKEY_BYTES` above.
+
+E.g.:
+
+```
+cargo test multiplier2 --release -- --nocapture
+cargo test keccak256 --release -- --nocapture
+cargo test rsa --release -- --nocapture
+```
+
+Will take corresponding `zkey` and put `arkzkey`` in same folder.
+
+## Multiplier
+
+NOTE: Need to change const ZKEY here
+
+`cargo test multiplier2 --release -- --nocapture`
+
+```
+running 1 test
+[build] Processing zkey data...
+[build] Time to process zkey data: 3.513041ms
+[build] Serializing proving key and constraint matrices
+[build] Time to serialize proving key and constraint matrices: 42ns
+[build] Writing arkzkey to: ../mopro-core/examples/circom/multiplier2/target/multiplier2_final.arkzkey
+[build] Time to write arkzkey: 1.884875ms
+Reading arkzkey from: ../mopro-core/examples/circom/multiplier2/target/multiplier2_final.arkzkey
+
+Time to open arkzkey file: 18.084µs
+Time to mmap arkzkey: 8.542µs
+Time to deserialize proving key: 305.75µs
+Time to deserialize matrices: 5µs
+Time to read arkzkey: 348.083µs
+test tests::test_multiplier2_serialization_deserialization ... ok
+```
+
+Naive test: `cargo test naive --release -- --nocapture` (with right zkey constant).
+
+**Result: `350µs` vs naive `3.3ms`**
+
+## Keccak
+
+NOTE: Need to change const ZKEY here
+
+`cargo test keccak256 --release -- --nocapture`
+
+```
+[build] Processing zkey data...
+test tests::test_keccak256_serialization_deserialization has been running for over 60 seconds
+[build]Time to process zkey data: 158.753181958s
+[build] Serializing proving key and constraint matrices
+[build] Time to serialize proving key and constraint matrices: 42ns
+[build] Writing arkzkey to: ../mopro-core/examples/circom/keccak256/target/keccak256_256_test_final.arkzkey
+[build] Time to write arkzkey: 16.204274125s
+Reading arkzkey from: ../mopro-core/examples/circom/keccak256/target/keccak256_256_test_final.arkzkey
+Time to open arkzkey file: 51.75µs
+Time to mmap arkzkey: 17.25µs
+Time to deserialize proving key: 18.323550083s
+Time to deserialize matrices: 46.935792ms
+Time to read arkzkey: 18.3730695s
+test tests::test_keccak256_serialization_deserialization ... ok
+```
+
+Vs naive:
+
+`[build] Time to process zkey data: 158.753181958s`
+
+
+**Result: 18s vs 158s**