diff --git a/Changelog.md b/Changelog.md index e127dbb..c087bd7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,13 @@ +# Version v1.3.0, 2020-10, Rust streaming + +Rust: +- Unified Workspace reader with streaming mode. +- Supports reading from stdin stream. +- Supports reading chunk by chunk from unlimited files. +- CLI stats, validate, and simulate work in streaming mode. +- *(breaking)* Renamed FileSink to WorkspaceSink. + + # Version v1.2.1, 2020-10 Rust: diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 92faf97..ffddd0f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "zkinterface" -version = "1.2.1" +version = "1.3.0" dependencies = [ "colored 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "flatbuffers 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b158211..6bdb375 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zkinterface" -version = "1.2.1" +version = "1.3.0" authors = ["Aurélien Nicolas "] license = "MIT" build = "build.rs" diff --git a/rust/src/cli.rs b/rust/src/cli.rs index 745bc7c..624f4aa 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -6,12 +6,7 @@ use std::io::{stdin, stdout, Read, Write, copy}; use std::path::{Path, PathBuf}; use structopt::StructOpt; -use crate::{ - Reader, - Messages, - consumers::stats::Stats, - Result, -}; +use crate::{Reader, Workspace, Messages, consumers::stats::Stats, Result}; use std::fs::{File, create_dir_all}; use std::ffi::OsStr; @@ -91,9 +86,9 @@ pub fn cli(options: &Options) -> Result<()> { "to-json" => main_json(&load_messages(options)?), "to-yaml" => main_yaml(&load_messages(options)?), "explain" => main_explain(&load_messages(options)?), - "validate" => main_validate(&load_messages(options)?), - "simulate" => main_simulate(&load_messages(options)?), - "stats" => main_stats(&load_messages(options)?), + "validate" => main_validate(&stream_messages(options)?), + "simulate" => main_simulate(&stream_messages(options)?), + "stats" => main_stats(&stream_messages(options)?), "fake_prove" => main_fake_prove(&load_messages(options)?), "fake_verify" => main_fake_verify(&load_messages(options)?), "help" => { @@ -116,7 +111,7 @@ fn load_messages(opts: &Options) -> Result { for path in list_files(opts)? { if path == Path::new("-") { eprintln!("Loading from stdin"); - reader.read_from(&mut stdin())?; + reader.read_from(stdin())?; } else { eprintln!("Loading file {}", path.display()); reader.read_file(path)?; @@ -127,6 +122,11 @@ fn load_messages(opts: &Options) -> Result { Ok(reader) } +fn stream_messages(opts: &Options) -> Result { + let paths = list_files(opts)?; + Ok(Workspace::new(paths)) +} + fn has_zkif_extension(path: &Path) -> bool { path.extension() == Some(OsStr::new("zkif")) } @@ -176,15 +176,17 @@ fn main_example(opts: &Options) -> Result<()> { } else { create_dir_all(out_dir)?; - let path = out_dir.join("statement.zkif"); - let mut file = File::create(&path)?; - example_circuit_header().write_into(&mut file)?; - example_constraints().write_into(&mut file)?; + let path = out_dir.join("header.zkif"); + example_circuit_header().write_into(&mut File::create(&path)?)?; eprintln!("Written {}", path.display()); let path = out_dir.join("witness.zkif"); example_witness().write_into(&mut File::create(&path)?)?; eprintln!("Written {}", path.display()); + + let path = out_dir.join("constraints.zkif"); + example_constraints().write_into(&mut File::create(&path)?)?; + eprintln!("Written {}", path.display()); } Ok(()) } @@ -217,25 +219,21 @@ fn main_explain(reader: &Reader) -> Result<()> { Ok(()) } -fn main_validate(reader: &Reader) -> Result<()> { - let reader = Messages::from(reader); - +fn main_validate(ws: &Workspace) -> Result<()> { // Validate semantics as verifier. let mut validator = Validator::new_as_verifier(); - validator.ingest_messages(&reader); + validator.ingest_workspace(ws); print_violations(&validator.get_violations()) } -fn main_simulate(reader: &Reader) -> Result<()> { - let reader = Messages::from(reader); - +fn main_simulate(ws: &Workspace) -> Result<()> { // Validate semantics as prover. let mut validator = Validator::new_as_prover(); - validator.ingest_messages(&reader); + validator.ingest_workspace(ws); print_violations(&validator.get_violations())?; // Check whether the statement is true. - let ok = Simulator::default().simulate(&reader); + let ok = Simulator::default().simulate_workspace(ws); match ok { Err(_) => eprintln!("The statement is NOT TRUE!"), Ok(_) => eprintln!("The statement is TRUE!"), @@ -254,9 +252,9 @@ fn print_violations(errors: &[String]) -> Result<()> { } } -fn main_stats(reader: &Reader) -> Result<()> { - let mut stats = Stats::new(); - stats.push(reader)?; +fn main_stats(ws: &Workspace) -> Result<()> { + let mut stats = Stats::default(); + stats.ingest_workspace(ws); serde_json::to_writer_pretty(stdout(), &stats)?; println!(); Ok(()) diff --git a/rust/src/consumers/mod.rs b/rust/src/consumers/mod.rs index 8144cbc..1c4c8ce 100644 --- a/rust/src/consumers/mod.rs +++ b/rust/src/consumers/mod.rs @@ -2,3 +2,4 @@ pub mod validator; pub mod simulator; pub mod stats; pub mod reader; +pub mod workspace; diff --git a/rust/src/consumers/reader.rs b/rust/src/consumers/reader.rs index be75d2a..0312d3f 100644 --- a/rust/src/consumers/reader.rs +++ b/rust/src/consumers/reader.rs @@ -62,7 +62,7 @@ pub fn split_messages(mut buf: &[u8]) -> Vec<&[u8]> { bufs } -pub fn read_buffer(stream: &mut impl Read) -> Result> { +pub fn read_buffer(mut stream: impl Read) -> Result> { let mut buffer = vec![0u8; 4]; if stream.read_exact(&mut buffer).is_err() { return Ok(Vec::new()); // End of stream at the correct place. @@ -164,9 +164,9 @@ impl Reader { Ok(()) } - pub fn read_from(&mut self, reader: &mut impl Read) -> Result<()> { + pub fn read_from(&mut self, mut reader: impl Read) -> Result<()> { loop { - let buffer = read_buffer(reader)?; + let buffer = read_buffer(&mut reader)?; if buffer.len() == 0 { return Ok(()); } diff --git a/rust/src/consumers/simulator.rs b/rust/src/consumers/simulator.rs index 64ba267..377aefc 100644 --- a/rust/src/consumers/simulator.rs +++ b/rust/src/consumers/simulator.rs @@ -1,4 +1,4 @@ -use crate::{Result, CircuitHeader, Witness, ConstraintSystem, Messages, Variables}; +use crate::{Result, CircuitHeader, Witness, ConstraintSystem, Messages, Variables, Workspace, Message}; use crate::structs::constraints::BilinearConstraint; use std::collections::HashMap; @@ -12,6 +12,8 @@ type Field = BigUint; pub struct Simulator { values: HashMap, modulus: Field, + + violations: Vec, } impl Simulator { @@ -28,6 +30,25 @@ impl Simulator { Ok(()) } + pub fn simulate_workspace(&mut self, ws: &Workspace) -> Result<()> { + for msg in ws.iter_messages() { + match msg { + Message::Header(header) => { + self.ingest_header(&header)?; + } + Message::ConstraintSystem(cs) => { + self.ingest_constraint_system(&cs)?; + } + Message::Witness(witness) => { + self.ingest_witness(&witness)?; + } + Message::Command(_) => {} + Message::Err(_) => {} + } + } + Ok(()) + } + pub fn ingest_header(&mut self, header: &CircuitHeader) -> Result<()> { // Set the field. let max = header.field_maximum.as_ref().ok_or("No field_maximum specified")?; diff --git a/rust/src/consumers/stats.rs b/rust/src/consumers/stats.rs index 5763023..b2357c3 100644 --- a/rust/src/consumers/stats.rs +++ b/rust/src/consumers/stats.rs @@ -3,10 +3,9 @@ extern crate serde_json; use serde::{Deserialize, Serialize}; -use crate::Reader; -use crate::Result; +use crate::{Workspace, Message}; -#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Default, Debug, Eq, PartialEq, Deserialize, Serialize)] pub struct Stats { num_public_inputs: u64, num_private_variables: u64, @@ -18,29 +17,38 @@ pub struct Stats { } impl Stats { - pub fn new() -> Stats { - return Stats { num_public_inputs: 0, num_private_variables: 0, multiplications: 0, additions_a: 0, additions_b: 0, additions_c: 0, additions: 0 }; - } + pub fn ingest_workspace(&mut self, ws: &Workspace) { + for msg in ws.iter_messages() { + match msg { + Message::Header(header) => { + self.num_public_inputs = header.instance_variables.variable_ids.len() as u64; + self.num_private_variables = header.free_variable_id - self.num_public_inputs - 1; + } - pub fn push(&mut self, reader: &Reader) -> Result<()> { - let header = reader.last_header().ok_or("no circuit")?; - self.num_public_inputs = header.instance_variables().unwrap().variable_ids().unwrap().len() as u64; - self.num_private_variables = header.free_variable_id() - self.num_public_inputs - 1; + Message::ConstraintSystem(cs) => { + self.multiplications += cs.constraints.len() as u64; - for constraint in reader.iter_constraints() { - self.multiplications += 1; - if constraint.a.len() > 0 { - self.additions_a += (constraint.a.len() - 1) as u64; - } - if constraint.b.len() > 0 { - self.additions_b += (constraint.b.len() - 1) as u64; - } - if constraint.c.len() > 0 { - self.additions_c += (constraint.c.len() - 1) as u64; + for constraint in &cs.constraints { + let len_a = constraint.linear_combination_a.variable_ids.len() as u64; + if len_a > 0 { + self.additions_a += len_a - 1; + } + + let len_b = constraint.linear_combination_b.variable_ids.len() as u64; + if len_b > 0 { + self.additions_b += len_b - 1; + } + + let len_c = constraint.linear_combination_c.variable_ids.len() as u64; + if len_c > 0 { + self.additions_c += len_c - 1; + } + } + self.additions = self.additions_a + self.additions_b + self.additions_c; + } + + _ => {} } } - - self.additions = self.additions_a + self.additions_b + self.additions_c; - Ok(()) } } diff --git a/rust/src/consumers/validator.rs b/rust/src/consumers/validator.rs index e5e67e8..32a0677 100644 --- a/rust/src/consumers/validator.rs +++ b/rust/src/consumers/validator.rs @@ -1,4 +1,4 @@ -use crate::{CircuitHeader, Witness, ConstraintSystem, Messages, Variables}; +use crate::{CircuitHeader, Witness, ConstraintSystem, Messages, Variables, Message, Workspace}; use std::collections::HashMap; use num_bigint::BigUint; @@ -51,8 +51,31 @@ impl Validator { } } + pub fn ingest_workspace(&mut self, ws: &Workspace) { + for msg in ws.iter_messages() { + match msg { + Message::Header(header) => { + self.ingest_header(&header); + } + Message::ConstraintSystem(cs) => { + self.ingest_constraint_system(&cs); + } + Message::Witness(witness) => { + if self.as_prover { + self.ingest_witness(&witness); + } + } + Message::Command(_) => {} + Message::Err(err) => self.violate(err.to_string()), + } + } + } + pub fn get_violations(mut self) -> Vec { self.ensure_all_variables_used(); + if !self.got_header { + self.violate("Missing header."); + } self.violations } diff --git a/rust/src/consumers/workspace.rs b/rust/src/consumers/workspace.rs new file mode 100644 index 0000000..319580e --- /dev/null +++ b/rust/src/consumers/workspace.rs @@ -0,0 +1,75 @@ +use std::path::{PathBuf, Path}; +use std::fs::File; +use std::iter; +use std::io::{Read, stdin}; +use crate::consumers::reader::read_buffer; +use crate::Message; + +pub struct Workspace { + paths: Vec, + stdin: bool, +} + +impl Workspace { + pub fn new(mut paths: Vec) -> Self { + if paths == vec![PathBuf::from("-")] { + Workspace { paths: vec![], stdin: true } + } else { + paths.sort(); + paths.sort_by_key(|path| { + let name = path.file_name().unwrap().to_str().unwrap(); + match () { + _ if name.contains("header") => 0, + _ if name.contains("witness") => 1, + _ if name.contains("constraint") => 3, + _ => 4, + } + }); + Workspace { paths, stdin: false } + } + } + + pub fn iter_messages<'w>(&'w self) -> impl Iterator + 'w { + let buffers: Box>> = if self.stdin { + Box::new(iterate_stream(stdin())) + } else { + Box::new(iterate_files(&self.paths)) + }; + + buffers.map(|buffer| Message::from(&buffer[..])) + } +} + +pub fn iterate_files<'w>(paths: &'w [PathBuf]) -> impl Iterator> + 'w { + paths.iter().flat_map(|path| + iterate_file(path)) +} + +pub fn iterate_file(path: &Path) -> Box>> { + match File::open(path) { + Err(err) => { + eprintln!("Error opening workspace file {}: {}", path.display(), err); + Box::new(iter::empty()) + } + Ok(file) => Box::new( + iterate_stream(file)), + } +} + +pub fn iterate_stream<'s>(mut stream: impl Read + 's) -> impl Iterator> + 's { + iter::from_fn(move || + match read_buffer(&mut stream) { + Err(err) => { + eprintln!("Error reading: {}", err); + None + } + Ok(buffer) => { + if buffer.len() == 0 { + None + } else { + Some(buffer) + } + } + } + ) +} \ No newline at end of file diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 5c0ae61..b62721d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -5,13 +5,21 @@ pub extern crate serde; pub mod zkinterface_generated; pub mod cli; -pub mod workspace; - -pub use consumers::reader::Reader; pub mod consumers; + +pub use consumers::{ + reader::Reader, + workspace::Workspace, +}; + pub mod producers; +pub use producers::{ + builder::{Sink, StatementBuilder}, + workspace::{WorkspaceSink, clean_workspace}, +}; + /// Fully-owned version of each data structure. /// These structures may be easier to work with than the /// no-copy versions found in zkinterface_generated and Reader. @@ -22,6 +30,7 @@ pub use structs::{ command::Command, constraints::{ConstraintSystem, BilinearConstraint}, keyvalue::KeyValue, + message::Message, messages::Messages, variables::Variables, witness::Witness, diff --git a/rust/src/producers/builder.rs b/rust/src/producers/builder.rs index 79e314e..3cb51b2 100644 --- a/rust/src/producers/builder.rs +++ b/rust/src/producers/builder.rs @@ -1,9 +1,5 @@ -use std::fs::{File, create_dir_all}; -use std::path::{Path, PathBuf}; - use crate::{Result, Variables, CircuitHeader, ConstraintSystem, Witness}; - pub trait Sink { fn push_header(&mut self, statement: CircuitHeader) -> Result<()>; fn push_constraints(&mut self, cs: ConstraintSystem) -> Result<()>; @@ -14,11 +10,10 @@ pub trait Sink { /// StatementBuilder assists with constructing and storing a statement in zkInterface format. /// # Example /// ``` -/// use zkinterface::producers::builder::{StatementBuilder, Sink, FileSink}; -/// use zkinterface::{CircuitHeader, ConstraintSystem, Witness}; +/// use zkinterface::{StatementBuilder, Sink, WorkspaceSink, CircuitHeader, ConstraintSystem, Witness}; /// /// // Create a workspace where to write zkInterafce files. -/// let sink = FileSink::new("local/test_builder").unwrap(); +/// let sink = WorkspaceSink::new("local/test_builder").unwrap(); /// let mut builder = StatementBuilder::new(sink); /// /// // Use variables, construct a constraint system, and a witness. @@ -85,56 +80,3 @@ impl Sink for StatementBuilder { fn push_constraints(&mut self, cs: ConstraintSystem) -> Result<()> { self.sink.push_constraints(cs) } fn push_witness(&mut self, witness: Witness) -> Result<()> { self.sink.push_witness(witness) } } - - -/// Store messages into files using conventional filenames inside of a workspace. -pub struct FileSink { - pub workspace: PathBuf, - pub constraints_file: Option, - pub witness_file: Option, -} - -impl FileSink { - pub fn new(workspace: impl AsRef) -> Result { - create_dir_all(workspace.as_ref())?; - Ok(FileSink { - workspace: workspace.as_ref().to_path_buf(), - constraints_file: None, - witness_file: None, - }) - } -} - -impl Sink for FileSink { - fn push_header(&mut self, header: CircuitHeader) -> Result<()> { - let mut file = File::create( - self.workspace.join("header.zkif"))?; - header.write_into(&mut file) - } - - fn push_constraints(&mut self, cs: ConstraintSystem) -> Result<()> { - let file = match self.constraints_file { - None => { - self.constraints_file = Some(File::create( - self.workspace.join("constraints.zkif"))?); - self.constraints_file.as_mut().unwrap() - } - Some(ref mut file) => file, - }; - - cs.write_into(file) - } - - fn push_witness(&mut self, witness: Witness) -> Result<()> { - let file = match self.witness_file { - None => { - self.witness_file = Some(File::create( - self.workspace.join("witness.zkif"))?); - self.witness_file.as_mut().unwrap() - } - Some(ref mut file) => file, - }; - - witness.write_into(file) - } -} diff --git a/rust/src/producers/gadget_caller.rs b/rust/src/producers/gadget_caller.rs index f217162..58668d8 100644 --- a/rust/src/producers/gadget_caller.rs +++ b/rust/src/producers/gadget_caller.rs @@ -1,5 +1,6 @@ use crate::{Result, CircuitHeader}; -use super::builder::{StatementBuilder, Sink, FileSink}; +use super::builder::{StatementBuilder, Sink}; +use super::workspace::WorkspaceSink; use std::io::Write; @@ -31,7 +32,7 @@ impl GadgetCallbacks for StatementBuilder { } } -impl GadgetCallbacks for FileSink { +impl GadgetCallbacks for WorkspaceSink { fn receive_constraints(&mut self, msg: &[u8]) -> Result<()> { if let Some(ref mut file) = self.constraints_file { file.write_all(msg)?; diff --git a/rust/src/producers/mod.rs b/rust/src/producers/mod.rs index 0a295a1..cf8e7fb 100644 --- a/rust/src/producers/mod.rs +++ b/rust/src/producers/mod.rs @@ -1,3 +1,4 @@ pub mod examples; pub mod builder; pub mod gadget_caller; +pub mod workspace; diff --git a/rust/src/producers/workspace.rs b/rust/src/producers/workspace.rs new file mode 100644 index 0000000..0a20902 --- /dev/null +++ b/rust/src/producers/workspace.rs @@ -0,0 +1,64 @@ +use std::path::{Path, PathBuf}; +use std::fs::{remove_file, File, create_dir_all}; +use crate::{Result, CircuitHeader, ConstraintSystem, Witness}; +use crate::producers::builder::Sink; + +pub fn clean_workspace(workspace: impl AsRef) -> Result<()> { + let workspace = workspace.as_ref(); + let _ = remove_file(workspace.join("header.zkif")); + let _ = remove_file(workspace.join("constraints.zkif")); + let _ = remove_file(workspace.join("witness.zkif")); + Ok(()) +} + +/// Store messages into files using conventional filenames inside of a workspace. +pub struct WorkspaceSink { + pub workspace: PathBuf, + pub constraints_file: Option, + pub witness_file: Option, +} + +impl WorkspaceSink { + pub fn new(workspace: impl AsRef) -> Result { + create_dir_all(workspace.as_ref())?; + Ok(WorkspaceSink { + workspace: workspace.as_ref().to_path_buf(), + constraints_file: None, + witness_file: None, + }) + } +} + +impl Sink for WorkspaceSink { + fn push_header(&mut self, header: CircuitHeader) -> Result<()> { + let mut file = File::create( + self.workspace.join("header.zkif"))?; + header.write_into(&mut file) + } + + fn push_constraints(&mut self, cs: ConstraintSystem) -> Result<()> { + let file = match self.constraints_file { + None => { + self.constraints_file = Some(File::create( + self.workspace.join("constraints.zkif"))?); + self.constraints_file.as_mut().unwrap() + } + Some(ref mut file) => file, + }; + + cs.write_into(file) + } + + fn push_witness(&mut self, witness: Witness) -> Result<()> { + let file = match self.witness_file { + None => { + self.witness_file = Some(File::create( + self.workspace.join("witness.zkif"))?); + self.witness_file.as_mut().unwrap() + } + Some(ref mut file) => file, + }; + + witness.write_into(file) + } +} diff --git a/rust/src/structs/message.rs b/rust/src/structs/message.rs new file mode 100644 index 0000000..ce9bf73 --- /dev/null +++ b/rust/src/structs/message.rs @@ -0,0 +1,39 @@ +use std::error::Error; +use crate::zkinterface_generated::zkinterface as fb; +use crate::{CircuitHeader, ConstraintSystem, Witness, Command}; + +pub enum Message { + Header(CircuitHeader), + ConstraintSystem(ConstraintSystem), + Witness(Witness), + Command(Command), + Err(Box), +} + +impl<'a> From<&'a [u8]> for Message { + fn from(buffer: &'a [u8]) -> Self { + let msg = fb::get_size_prefixed_root_as_root(&buffer); + + match msg.message_type() { + fb::Message::CircuitHeader => { + let fb_header = msg.message_as_circuit_header().unwrap(); + Message::Header(CircuitHeader::from(fb_header)) + } + fb::Message::ConstraintSystem => { + let fb_constraints = msg.message_as_constraint_system().unwrap(); + Message::ConstraintSystem(ConstraintSystem::from(fb_constraints)) + } + fb::Message::Witness => { + let fb_witness = msg.message_as_witness().unwrap(); + Message::Witness(Witness::from(fb_witness)) + } + fb::Message::Command => { + let fb_command = msg.message_as_command().unwrap(); + Message::Command(Command::from(fb_command)) + } + fb::Message::NONE => { + Message::Err("Invalid message type".into()) + } + } + } +} \ No newline at end of file diff --git a/rust/src/structs/mod.rs b/rust/src/structs/mod.rs index 0eb1b82..e499ea8 100644 --- a/rust/src/structs/mod.rs +++ b/rust/src/structs/mod.rs @@ -1,3 +1,4 @@ +pub mod message; pub mod messages; pub mod header; pub mod command; diff --git a/rust/src/workspace.rs b/rust/src/workspace.rs deleted file mode 100644 index a3681aa..0000000 --- a/rust/src/workspace.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::path::Path; -use std::fs::remove_file; -use crate::Result; - -pub fn clean_workspace(workspace: impl AsRef) -> Result<()> { - let workspace = workspace.as_ref(); - let _ = remove_file(workspace.join("header.zkif")); - let _ = remove_file(workspace.join("constraints.zkif")); - let _ = remove_file(workspace.join("witness.zkif")); - Ok(()) -}