diff --git a/src/dynamic_merkle_tree.rs b/src/dynamic_merkle_tree.rs index cf62382..0510238 100644 --- a/src/dynamic_merkle_tree.rs +++ b/src/dynamic_merkle_tree.rs @@ -1,8 +1,5 @@ -use crate::{ - merkle_tree::{Branch, Hasher, Proof}, - util::as_bytes, -}; -use color_eyre::eyre::Result; +use crate::merkle_tree::{Branch, Hasher, Proof}; +use color_eyre::eyre::{bail, Result}; use std::{ fs::OpenOptions, io::Write, @@ -39,19 +36,24 @@ pub struct DynamicMerkleTree = Vec<> DynamicMerkleTree { #[must_use] - pub fn new(depth: usize, empty_value: &H::Hash) -> DynamicMerkleTree { - Self::new_with_leaves(depth, empty_value, &[]) + pub fn new( + config: S::StorageConfig, + depth: usize, + empty_value: &H::Hash, + ) -> DynamicMerkleTree { + Self::new_with_leaves(config, depth, empty_value, &[]) } /// initial leaves populated from the given slice. #[must_use] pub fn new_with_leaves( + config: S::StorageConfig, depth: usize, empty_value: &H::Hash, leaves: &[H::Hash], ) -> DynamicMerkleTree { assert!(depth > 0); - let storage = Self::storage_from_leaves(empty_value, leaves); + let storage = Self::storage_from_leaves(config, empty_value, leaves); let sparse_column = Self::sparse_column(depth, empty_value); let num_leaves = leaves.len(); @@ -81,7 +83,11 @@ impl> DynamicMerkleTree { .collect() } - fn storage_from_leaves(empty_value: &H::Hash, leaves: &[H::Hash]) -> S { + fn storage_from_leaves( + config: S::StorageConfig, + empty_value: &H::Hash, + leaves: &[H::Hash], + ) -> S { let num_leaves = leaves.len(); let base_len = num_leaves.next_power_of_two(); let storage_size = base_len << 1; @@ -103,7 +109,7 @@ impl> DynamicMerkleTree { last_sub_root = hash; } - S::checked_from_vec(storage).unwrap() + S::init(config, storage).unwrap() } /// Subtrees are 1 indexed and directly attached to the left most branch @@ -157,17 +163,19 @@ impl> DynamicMerkleTree { // Self::init_subtree(&self.sparse_column, &mut // self.storage[current_size..]); } - pub fn push(&mut self, leaf: H::Hash) { + pub fn push(&mut self, leaf: H::Hash) -> Result<()> { let index = index_from_leaf(self.num_leaves); match self.storage.get_mut(index) { Some(val) => *val = leaf, None => { - self.storage.reallocate(&self.sparse_column); + self.storage + .reallocate(&self.empty_value, &self.sparse_column)?; self.storage[index] = leaf; } } self.num_leaves += 1; self.propogate_up(index); + Ok(()) } // pub fn extend_from_slice(&mut self, leaves: &[H::Hash]) { @@ -311,11 +319,13 @@ impl> DynamicMerkleTree { // so that we can have type level information that the length // is always exactly a power of 2 pub trait DynamicTreeStorage: - Deref + DerefMut + Default + Deref + DerefMut + Sized { - fn reallocate(&mut self, sparse_column: &[H::Hash]); + type StorageConfig; - fn checked_from_vec(vec: Vec) -> Result; + fn reallocate(&mut self, empty_leaf: &H::Hash, sparse_column: &[H::Hash]) -> Result<()>; + + fn init(config: Self::StorageConfig, vec: Vec) -> Result; fn storage_root(&self) -> H::Hash { self[self.len() >> 1] @@ -327,153 +337,199 @@ pub trait DynamicTreeStorage: } impl DynamicTreeStorage for Vec { - fn reallocate(&mut self, sparse_column: &[H::Hash]) { + type StorageConfig = (); + + fn init(_config: (), vec: Vec) -> Result { + Ok(vec) + } + + fn reallocate(&mut self, empty_leaf: &H::Hash, sparse_column: &[H::Hash]) -> Result<()> { let current_size = self.len(); - self.extend(repeat(self[0]).take(current_size)); + self.extend(repeat(*empty_leaf).take(current_size)); init_subtree::(sparse_column, &mut self[current_size..]); + Ok(()) } +} - fn checked_from_vec(vec: Vec) -> Result { - Ok(vec) +pub struct MmapTreeStorageConfig { + pub file_path: PathBuf, +} + +impl DynamicTreeStorage for MmapVec { + type StorageConfig = MmapTreeStorageConfig; + + fn init(config: MmapTreeStorageConfig, vec: Vec) -> Result { + Ok(Self::new(config.file_path, &vec)?) + } + + fn reallocate(&mut self, empty_leaf: &H::Hash, sparse_column: &[H::Hash]) -> Result<()> { + let current_size = self.len(); + self.reallocate(empty_leaf)?; + init_subtree::(sparse_column, &mut self[current_size..]); + Ok(()) + } +} + +pub struct MmapVec { + mmap: MmapMut, + file_path: PathBuf, + phantom: std::marker::PhantomData, +} + +impl MmapVec { + /// Creates a new memory map backed with file with provided size + /// and fills the entire map with initial value + /// + /// # Errors + /// + /// - returns Err if file creation has failed + /// - returns Err if bytes couldn't be written to file + /// + /// # Panics + /// + /// - empty hash value serialization failed + /// - file size cannot be set + /// - file is too large, possible truncation can occur + /// - cannot build memory map + pub fn new(file_path: PathBuf, storage: &[H::Hash]) -> Result { + // Safety: potential uninitialized padding from `H::Hash` is safe to use if + // we're casting back to the same type. + let buf = bytemuck::cast_slice(storage); + let buf_len = buf.len(); + + let mut file = match OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(file_path.clone()) + { + Ok(file) => file, + Err(_e) => return Err(DenseMMapError::FileCreationFailed), + }; + + file.set_len(buf_len as u64).expect("cannot set file size"); + if file.write_all(buf).is_err() { + return Err(DenseMMapError::FileCannotWriteBytes); + } + + let mmap = unsafe { + MmapOptions::new(usize::try_from(buf_len as u64).expect("file size truncated")) + .expect("cannot create memory map") + .with_file(file, 0) + .map_mut() + .expect("cannot build memory map") + }; + + Ok(Self { + mmap, + file_path, + phantom: std::marker::PhantomData, + }) + } + + /// Given the file path and tree depth, + /// it attempts to restore the memory map + /// + /// # Errors + /// + /// - returns Err if file doesn't exist + /// - returns Err if file size doesn't match the expected tree size + /// + /// # Panics + /// + /// - cannot get file metadata to check for file length + /// - truncated file size when attempting to build memory map + /// - cannot build memory map + pub fn restore(empty_leaf: &H::Hash, file_path: PathBuf) -> Result { + let file = match OpenOptions::new() + .read(true) + .write(true) + .open(file_path.clone()) + { + Ok(file) => file, + Err(_e) => bail!("File doesn't exist"), + }; + + let file_size = file.metadata().expect("cannot get file metadata").len(); + let size_of_empty_leaf = std::mem::size_of_val(empty_leaf); + if !(file_size / size_of_empty_leaf as u64).is_power_of_two() { + bail!("File size should be a power of 2"); + } + + let mmap = unsafe { + MmapOptions::new(file_size as usize) + .expect("cannot create memory map") + .with_file(file, 0) + .map_mut() + .expect("cannot build memory map") + }; + + Ok(Self { + mmap, + file_path, + phantom: std::marker::PhantomData, + }) + } + + pub fn reallocate(&mut self, empty_leaf: &H::Hash) -> Result<()> { + let file = match OpenOptions::new() + .read(true) + .write(true) + .open(self.file_path.clone()) + { + Ok(file) => file, + Err(_e) => bail!("File doesn't exist"), + }; + + let file_size = file.metadata().expect("cannot get file metadata").len(); + let size_of_empty_leaf = std::mem::size_of_val(empty_leaf); + if !(file_size / size_of_empty_leaf as u64).is_power_of_two() { + bail!("File size should be a power of 2"); + } + + let new_file_size = file_size << 1; + file.set_len(new_file_size).expect("cannot expand size"); + + self.mmap = unsafe { + MmapOptions::new(new_file_size as usize) + .expect("cannot create memory map") + .with_file(file, 0) + .map_mut() + .expect("cannot build memory map") + }; + + Ok(()) } } -// pub struct DynamicMMapMerkleTree(MmapMutWrapper); -// -// pub struct MmapMutWrapper { -// mmap: MmapMut, -// phantom: std::marker::PhantomData, -// } -// -// impl MmapMutWrapper { -// /// Creates a new memory map backed with file with provided size -// /// and fills the entire map with initial value -// /// -// /// # Errors -// /// -// /// - returns Err if file creation has failed -// /// - returns Err if bytes couldn't be written to file -// /// -// /// # Panics -// /// -// /// - empty hash value serialization failed -// /// - file size cannot be set -// /// - file is too large, possible truncation can occur -// /// - cannot build memory map -// pub fn new( -// file_path: PathBuf, -// dynamic_tree: &DynamicMerkleTree, -// ) -> Result { -// // Safety: potential uninitialized padding from `H::Hash` is safe to -// use if // we're casting back to the same type. -// let buf: &[u8] = unsafe { as_bytes(&dynamic_tree) }; -// let buf_len = buf.len(); -// -// let mut file = match OpenOptions::new() -// .read(true) -// .write(true) -// .create(true) -// .truncate(true) -// .open(file_path) -// { -// Ok(file) => file, -// Err(_e) => return Err(DenseMMapError::FileCreationFailed), -// }; -// -// file.set_len(buf_len as u64).expect("cannot set file size"); -// if file.write_all(buf).is_err() { -// return Err(DenseMMapError::FileCannotWriteBytes); -// } -// -// let mmap = unsafe { -// MmapOptions::new(usize::try_from(buf_len as u64).expect("file -// size truncated")) .expect("cannot create memory map") -// .with_file(file, 0) -// .map_mut() -// .expect("cannot build memory map") -// }; -// -// Ok(Self { -// mmap, -// phantom: std::marker::PhantomData, -// }) -// } -// -// /// Given the file path and tree depth, -// /// it attempts to restore the memory map -// /// -// /// # Errors -// /// -// /// - returns Err if file doesn't exist -// /// - returns Err if file size doesn't match the expected tree size -// /// -// /// # Panics -// /// -// /// - cannot get file metadata to check for file length -// /// - truncated file size when attempting to build memory map -// /// - cannot build memory map -// pub fn attempt_restore( -// empty_leaf: &H::Hash, -// depth: usize, -// file_path: PathBuf, -// ) -> Result { -// let file = match -// OpenOptions::new().read(true).write(true).open(file_path) { -// Ok(file) => file, Err(_e) => return -// Err(DenseMMapError::FileDoesntExist), }; -// -// let size_of_empty_leaf = std::mem::size_of_val(empty_leaf); -// let expected_file_size = (1 << (depth + 1)) * size_of_empty_leaf as -// u64; -// -// if expected_file_size != file.metadata().expect("cannot get file -// metadata").len() { return -// Err(DenseMMapError::FileSizeShouldMatchTree); } -// -// let mmap = unsafe { -// MmapOptions::new( -// usize::try_from(expected_file_size).expect("expected file -// size truncated"), ) -// .expect("cannot create memory map") -// .with_file(file, 0) -// .map_mut() -// .expect("cannot build memory map") -// }; -// -// Ok(Self { -// mmap, -// phantom: std::marker::PhantomData, -// }) -// } -// } -// -// impl Deref for MmapMutWrapper { -// type Target = [H::Hash]; -// -// fn deref(&self) -> &Self::Target { -// bytemuck::cast_slice(self.mmap.as_slice()) -// } -// } -// -// impl DerefMut for MmapMutWrapper { -// fn deref_mut(&mut self) -> &mut Self::Target { -// bytemuck::cast_slice_mut(self.mmap.as_mut_slice()) -// } -// } -// -// #[derive(Error, Debug)] -// pub enum DenseMMapError { -// #[error("file size should match expected tree size")] -// FileSizeShouldMatchTree, -// #[error("file doesn't exist")] -// FileDoesntExist, -// #[error("failed to create a file")] -// FileCreationFailed, -// #[error("cannot write bytes to file")] -// FileCannotWriteBytes, -// #[error("failed to create pathbuf")] -// FailedToCreatePathBuf, -// } +impl Deref for MmapVec { + type Target = [H::Hash]; + + fn deref(&self) -> &Self::Target { + bytemuck::cast_slice(self.mmap.as_slice()) + } +} + +impl DerefMut for MmapVec { + fn deref_mut(&mut self) -> &mut Self::Target { + bytemuck::cast_slice_mut(self.mmap.as_mut_slice()) + } +} + +#[derive(Error, Debug)] +pub enum DenseMMapError { + #[error("file size should match expected tree size")] + FileSizeShouldMatchTree, + #[error("file doesn't exist")] + FileDoesntExist, + #[error("failed to create a file")] + FileCreationFailed, + #[error("cannot write bytes to file")] + FileCannotWriteBytes, + #[error("failed to create pathbuf")] + FailedToCreatePathBuf, +} // leaves are 0 indexed fn index_from_leaf(leaf: usize) -> usize { @@ -508,7 +564,7 @@ fn sibling(i: usize) -> Branch { } } -fn children(i: usize) -> Option<(usize, usize)> { +fn _children(i: usize) -> Option<(usize, usize)> { let next_pow = i.next_power_of_two(); if i == next_pow { if i == 1 { @@ -584,7 +640,7 @@ mod tests { for _ in 1..storage_depth { let next = previous .iter() - .flat_map(|&i| children(i)) + .flat_map(|&i| _children(i)) .collect::>(); previous = next.iter().flat_map(|&(l, r)| [l, r]).collect(); let row = previous @@ -665,7 +721,7 @@ mod tests { fn test_children() { let mut children = Vec::new(); for i in 1..16 { - children.push((i, super::children(i))); + children.push((i, super::_children(i))); } let expected_siblings = vec![ (1, None), @@ -693,7 +749,7 @@ mod tests { let num_leaves = 1; let leaves = vec![1; num_leaves]; let empty = 0; - let tree = DynamicMerkleTree::::new_with_leaves(1, &empty, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 1, &empty, &leaves); debug_tree(&tree); } @@ -703,7 +759,7 @@ mod tests { let num_leaves = 1; let leaves = vec![1; num_leaves]; let empty = 0; - let tree = DynamicMerkleTree::::new_with_leaves(0, &empty, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 0, &empty, &leaves); debug_tree(&tree); } @@ -711,7 +767,7 @@ mod tests { fn test_odd_leaves() { let num_leaves = 5; let leaves = vec![1; num_leaves]; - let tree = DynamicMerkleTree::::new_with_leaves(10, &0, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 10, &0, &leaves); let expected = DynamicMerkleTree:: { depth: 10, num_leaves: 5, @@ -730,7 +786,7 @@ mod tests { let num_leaves = 1 << 3; let leaves = vec![1; num_leaves]; let empty = 0; - let tree = DynamicMerkleTree::::new_with_leaves(10, &empty, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 10, &empty, &leaves); let expected = DynamicMerkleTree:: { depth: 10, num_leaves: 8, @@ -748,7 +804,7 @@ mod tests { fn test_no_leaves() { let leaves = vec![]; let empty = 0; - let tree = DynamicMerkleTree::::new_with_leaves(10, &empty, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 10, &empty, &leaves); let expected = DynamicMerkleTree:: { depth: 10, num_leaves: 0, @@ -766,7 +822,7 @@ mod tests { fn test_sparse_column() { let leaves = vec![]; let empty = 1; - let tree = DynamicMerkleTree::::new_with_leaves(10, &empty, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 10, &empty, &leaves); let expected = DynamicMerkleTree:: { depth: 10, num_leaves: 0, @@ -785,7 +841,7 @@ mod tests { let num_leaves = 1 << 3; let leaves = vec![0; num_leaves]; let empty = 1; - let tree = DynamicMerkleTree::::new_with_leaves(4, &empty, &leaves); + let tree = DynamicMerkleTree::::new_with_leaves((), 4, &empty, &leaves); let expected = DynamicMerkleTree:: { depth: 4, num_leaves: 8, @@ -804,9 +860,9 @@ mod tests { let num_leaves = 1 << 3; let leaves = vec![1; num_leaves]; let empty = 0; - let mut tree = DynamicMerkleTree::::new_with_leaves(10, &empty, &leaves); + let mut tree = DynamicMerkleTree::::new_with_leaves((), 10, &empty, &leaves); debug_tree(&tree); - tree.push(3); + tree.push(3).unwrap(); debug_tree(&tree); } } diff --git a/src/lazy_merkle_tree.rs b/src/lazy_merkle_tree.rs index adb5822..24aa669 100644 --- a/src/lazy_merkle_tree.rs +++ b/src/lazy_merkle_tree.rs @@ -1,7 +1,4 @@ -use crate::{ - merkle_tree::{Branch, Hasher, Proof}, - util::as_bytes, -}; +use crate::merkle_tree::{Branch, Hasher, Proof}; use std::{ fs::OpenOptions, io::Write, @@ -246,11 +243,8 @@ impl AnyTree { let mut result: Self = dense.into(); let mut current_depth = prefix_depth; while current_depth < depth { - result = SparseTree::new( - result, - EmptyTree::new(current_depth, empty_value.clone()).into(), - ) - .into(); + result = + SparseTree::new(result, EmptyTree::new(current_depth, *empty_value).into()).into(); current_depth += 1; } result @@ -258,16 +252,13 @@ impl AnyTree { fn new_with_dense_prefix(depth: usize, prefix_depth: usize, empty_value: &H::Hash) -> Self { assert!(depth >= prefix_depth); - let mut result: Self = EmptyTree::new(prefix_depth, empty_value.clone()) + let mut result: Self = EmptyTree::new(prefix_depth, *empty_value) .alloc_dense() .into(); let mut current_depth = prefix_depth; while current_depth < depth { - result = SparseTree::new( - result, - EmptyTree::new(current_depth, empty_value.clone()).into(), - ) - .into(); + result = + SparseTree::new(result, EmptyTree::new(current_depth, *empty_value).into()).into(); current_depth += 1; } result @@ -286,11 +277,8 @@ impl AnyTree { let mut result: Self = dense.into(); let mut current_depth = prefix_depth; while current_depth < depth { - result = SparseTree::new( - result, - EmptyTree::new(current_depth, empty_value.clone()).into(), - ) - .into(); + result = + SparseTree::new(result, EmptyTree::new(current_depth, *empty_value).into()).into(); current_depth += 1; } Ok(result) @@ -308,11 +296,8 @@ impl AnyTree { let mut current_depth = prefix_depth; while current_depth < depth { - result = SparseTree::new( - result, - EmptyTree::new(current_depth, empty_leaf.clone()).into(), - ) - .into(); + result = + SparseTree::new(result, EmptyTree::new(current_depth, *empty_leaf).into()).into(); current_depth += 1; } @@ -457,7 +442,7 @@ impl EmptyTree { fn write_proof(&self, index: usize, path: &mut Vec>) { for depth in (1..=self.depth).rev() { - let val = self.empty_tree_values[depth - 1].clone(); + let val = self.empty_tree_values[depth - 1]; let branch = if get_turn_at_depth(index, depth) == Turn::Left { Branch::Left(val) } else { @@ -512,11 +497,11 @@ impl EmptyTree { #[must_use] fn root(&self) -> H::Hash { - self.empty_tree_values[self.depth].clone() + self.empty_tree_values[self.depth] } fn get_leaf(&self) -> H::Hash { - self.empty_tree_values[0].clone() + self.empty_tree_values[0] } } @@ -580,7 +565,7 @@ impl Clone for SparseTree { fn clone(&self) -> Self { Self { depth: self.depth, - root: self.root.clone(), + root: self.root, children: self.children.clone(), } } @@ -626,7 +611,7 @@ impl SparseTree { ) -> Self { let Some(children) = &self.children else { // no children – this is a leaf - return Self::new_leaf(value.clone()); + return Self::new_leaf(*value); }; let next_index = clear_turn_at_depth(index, self.depth); @@ -652,12 +637,12 @@ impl SparseTree { } fn root(&self) -> H::Hash { - self.root.clone() + self.root } fn get_leaf(&self, index: usize) -> H::Hash { self.children.as_ref().map_or_else( - || self.root.clone(), + || self.root, |children| { let next_index = clear_turn_at_depth(index, self.depth); if get_turn_at_depth(index, self.depth) == Turn::Left { @@ -693,11 +678,11 @@ impl DenseTree { let storage_size = 1 << (depth + 1); let mut storage = Vec::with_capacity(storage_size); - let empties = repeat(empty_value.clone()).take(leaf_count); + let empties = repeat(empty_value).take(leaf_count); storage.extend(empties); storage.extend_from_slice(values); if values.len() < leaf_count { - let empties = repeat(empty_value.clone()).take(leaf_count - values.len()); + let empties = repeat(empty_value).take(leaf_count - values.len()); storage.extend(empties); } @@ -750,7 +735,7 @@ impl DenseTree { fn get_leaf(&self, index: usize) -> H::Hash { self.with_ref(|r| { let leaf_index_in_dense_tree = index + (self.root_index << self.depth); - r.storage[leaf_index_in_dense_tree].clone() + r.storage[leaf_index_in_dense_tree] }) } @@ -771,7 +756,7 @@ impl DenseTree { fn update_with_mutation(&self, index: usize, value: &H::Hash) { let mut storage = self.storage.lock().expect("lock poisoned, terminating"); let leaf_index_in_dense_tree = index + (self.root_index << self.depth); - storage[leaf_index_in_dense_tree] = value.clone(); + storage[leaf_index_in_dense_tree] = *value; let mut current = leaf_index_in_dense_tree / 2; while current > 0 { let left = &storage[2 * current]; diff --git a/src/merkle_tree.rs b/src/merkle_tree.rs index aadd7d9..f700fe9 100644 --- a/src/merkle_tree.rs +++ b/src/merkle_tree.rs @@ -120,7 +120,7 @@ impl MerkleTree { #[must_use] pub fn root(&self) -> H::Hash { - self.nodes[0].clone() + self.nodes[0] } pub fn set(&mut self, leaf: usize, hash: H::Hash) {