From 15b096a765dc09aed11cdde52f76990381d6acee Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 17 Mar 2023 09:30:22 +0000 Subject: [PATCH 1/6] Use libfuzzer-sys from crates.io --- trie-db/fuzz/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index 225de3be..889212f0 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -17,7 +17,7 @@ reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0 path = ".." [dependencies.libfuzzer-sys] -git = "https://github.com/rust-fuzz/libfuzzer-sys.git" +version = "0.4.6" # Prevent this from interfering with workspaces [workspace] From 7fa284f555e7778a8cf7f16437e4f50fb6d0d0f0 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 17 Mar 2023 09:31:49 +0000 Subject: [PATCH 2/6] Copy-paste substrate's trie layout into the reference trie --- test-support/reference-trie/Cargo.toml | 1 + test-support/reference-trie/src/lib.rs | 20 + test-support/reference-trie/src/substrate.rs | 725 +++++++++++++++++++ 3 files changed, 746 insertions(+) create mode 100644 test-support/reference-trie/src/substrate.rs diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index cb9e5e05..93404dd8 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -14,6 +14,7 @@ trie-db = { path = "../../trie-db", default-features = false, version = "0.27.0" trie-root = { path = "../../trie-root", default-features = false, version = "0.18.0" } parity-scale-codec = { version = "3.0.0", features = ["derive"] } hashbrown = { version = "0.13.2", default-features = false, features = ["ahash"] } +paste = "1.0.12" [dev-dependencies] trie-bench = { path = "../trie-bench" } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 84448c98..be626801 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -28,6 +28,7 @@ use trie_db::{ pub use trie_root::TrieStream; use trie_root::{Hasher, Value as TrieStreamValue}; +mod substrate; mod substrate_like; pub mod node { pub use trie_db::node::Node; @@ -38,6 +39,9 @@ pub use substrate_like::{ NodeCodec as ReferenceNodeCodecNoExtMeta, ReferenceTrieStreamNoExt, }; +pub use paste::paste; +pub use substrate::{LayoutV0 as SubstrateV0, LayoutV1 as SubstrateV1}; + /// Reference hasher is a keccak hasher. pub type RefHasher = keccak_hasher::KeccakHasher; @@ -59,6 +63,22 @@ macro_rules! test_layouts { }; } +#[macro_export] +macro_rules! test_layouts_substrate { + ($test:ident) => { + $crate::paste! { + #[test] + fn [<$test _substrate_v0>]() { + $test::<$crate::SubstrateV0<$crate::RefHasher>>(); + } + #[test] + fn [<$test _substrate_v1>]() { + $test::<$crate::SubstrateV1<$crate::RefHasher>>(); + } + } + }; +} + /// Apply a test method on every test layouts. #[macro_export] macro_rules! test_layouts_no_meta { diff --git a/test-support/reference-trie/src/substrate.rs b/test-support/reference-trie/src/substrate.rs new file mode 100644 index 00000000..9b1573f1 --- /dev/null +++ b/test-support/reference-trie/src/substrate.rs @@ -0,0 +1,725 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Codec and layout directly copy-pasted from substrate with minimal modifications. + +use core::{borrow::Borrow, iter::once, marker::PhantomData, ops::Range}; +use hash_db::Hasher; +use parity_scale_codec as codec; +use parity_scale_codec::{Compact, Decode, Encode, Input, Output}; +use trie_db::{ + nibble_ops, + node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, + ChildReference, NodeCodec as NodeCodecT, TrieConfiguration, TrieLayout, +}; + +/// Constants used into trie simplification codec. +mod trie_constants { + const FIRST_PREFIX: u8 = 0b_00 << 6; + pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; + pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; + pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; + pub const EMPTY_TRIE: u8 = FIRST_PREFIX | (0b_00 << 4); + pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_1 << 5); + pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); + pub const ESCAPE_COMPACT_HEADER: u8 = EMPTY_TRIE | 0b_00_01; +} + +pub const TRIE_VALUE_NODE_THRESHOLD: u32 = 33; + +/// Codec-flavored TrieStream. +#[derive(Default, Clone)] +pub struct TrieStream { + /// Current node buffer. + buffer: Vec, +} + +fn branch_node_bit_mask(has_children: impl Iterator) -> (u8, u8) { + let mut bitmap: u16 = 0; + let mut cursor: u16 = 1; + for v in has_children { + if v { + bitmap |= cursor + } + cursor <<= 1; + } + ((bitmap % 256) as u8, (bitmap / 256) as u8) +} + +/// Create a leaf/branch node, encoding a number of nibbles. +fn fuse_nibbles_node(nibbles: &[u8], kind: NodeKind) -> impl Iterator + '_ { + let size = nibbles.len(); + let iter_start = match kind { + NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), + NodeKind::BranchNoValue => + size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), + NodeKind::BranchWithValue => + size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), + NodeKind::HashedValueLeaf => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), + NodeKind::HashedValueBranch => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), + }; + iter_start + .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) + .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) +} + +use trie_root::Value as TrieStreamValue; +impl trie_root::TrieStream for TrieStream { + fn new() -> Self { + Self { buffer: Vec::new() } + } + + fn append_empty_data(&mut self) { + self.buffer.push(trie_constants::EMPTY_TRIE); + } + + fn append_leaf(&mut self, key: &[u8], value: TrieStreamValue) { + let kind = match &value { + TrieStreamValue::Inline(..) => NodeKind::Leaf, + TrieStreamValue::Node(..) => NodeKind::HashedValueLeaf, + }; + self.buffer.extend(fuse_nibbles_node(key, kind)); + match &value { + TrieStreamValue::Inline(value) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + TrieStreamValue::Node(hash) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, + }; + } + + fn begin_branch( + &mut self, + maybe_partial: Option<&[u8]>, + maybe_value: Option, + has_children: impl Iterator, + ) { + if let Some(partial) = maybe_partial { + let kind = match &maybe_value { + None => NodeKind::BranchNoValue, + Some(TrieStreamValue::Inline(..)) => NodeKind::BranchWithValue, + Some(TrieStreamValue::Node(..)) => NodeKind::HashedValueBranch, + }; + + self.buffer.extend(fuse_nibbles_node(partial, kind)); + let bm = branch_node_bit_mask(has_children); + self.buffer.extend([bm.0, bm.1].iter()); + } else { + unreachable!("trie stream codec only for no extension trie"); + } + match maybe_value { + None => (), + Some(TrieStreamValue::Inline(value)) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + Some(TrieStreamValue::Node(hash)) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, + } + } + + fn append_extension(&mut self, _key: &[u8]) { + debug_assert!(false, "trie stream codec only for no extension trie"); + } + + fn append_substream(&mut self, other: Self) { + let data = other.out(); + match data.len() { + 0..=31 => data.encode_to(&mut self.buffer), + _ => H::hash(&data).as_ref().encode_to(&mut self.buffer), + } + } + + fn out(self) -> Vec { + self.buffer + } +} + +/// Helper struct for trie node decoder. This implements `codec::Input` on a byte slice, while +/// tracking the absolute position. This is similar to `std::io::Cursor` but does not implement +/// `Read` and `io` is not in `sp-std`. +struct ByteSliceInput<'a> { + data: &'a [u8], + offset: usize, +} + +impl<'a> ByteSliceInput<'a> { + fn new(data: &'a [u8]) -> Self { + ByteSliceInput { data, offset: 0 } + } + + fn take(&mut self, count: usize) -> Result, codec::Error> { + if self.offset + count > self.data.len() { + return Err("out of data".into()) + } + + let range = self.offset..(self.offset + count); + self.offset += count; + Ok(range) + } +} + +impl<'a> Input for ByteSliceInput<'a> { + fn remaining_len(&mut self) -> Result, codec::Error> { + Ok(Some(self.data.len().saturating_sub(self.offset))) + } + + fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { + let range = self.take(into.len())?; + into.copy_from_slice(&self.data[range]); + Ok(()) + } + + fn read_byte(&mut self) -> Result { + if self.offset + 1 > self.data.len() { + return Err("out of data".into()) + } + + let byte = self.data[self.offset]; + self.offset += 1; + Ok(byte) + } +} + +/// Concrete implementation of a [`NodeCodecT`] with SCALE encoding. +/// +/// It is generic over `H` the [`Hasher`]. +#[derive(Default, Clone)] +pub struct NodeCodec(PhantomData); + +impl NodeCodecT for NodeCodec +where + H: Hasher, +{ + const ESCAPE_HEADER: Option = Some(trie_constants::ESCAPE_COMPACT_HEADER); + type Error = Error; + type HashOut = H::Out; + + fn hashed_null_node() -> ::Out { + H::hash(::empty_node()) + } + + fn decode_plan(data: &[u8]) -> Result { + let mut input = ByteSliceInput::new(data); + + let header = NodeHeader::decode(&mut input)?; + let contains_hash = header.contains_hash_of_value(); + + let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { + *has_value + } else { + // hashed_value_branch + true + }; + + match header { + NodeHeader::Null => Ok(NodePlan::Empty), + NodeHeader::HashedValueBranch(nibble_count) | NodeHeader::Branch(_, nibble_count) => { + let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; + // check that the padding is valid (if any) + if padding && nibble_ops::pad_left(data[input.offset]) != 0 { + return Err(Error::BadFormat) + } + let partial = input.take( + (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / + nibble_ops::NIBBLE_PER_BYTE, + )?; + let partial_padding = nibble_ops::number_padding(nibble_count); + let bitmap_range = input.take(BITMAP_LENGTH)?; + let bitmap = Bitmap::decode(&data[bitmap_range])?; + let value = if branch_has_value { + Some(if contains_hash { + ValuePlan::Node(input.take(H::LENGTH)?) + } else { + let count = >::decode(&mut input)?.0 as usize; + ValuePlan::Inline(input.take(count)?) + }) + } else { + None + }; + let mut children = [ + None, None, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, + ]; + for i in 0..nibble_ops::NIBBLE_LENGTH { + if bitmap.value_at(i) { + let count = >::decode(&mut input)?.0 as usize; + let range = input.take(count)?; + children[i] = Some(if count == H::LENGTH { + NodeHandlePlan::Hash(range) + } else { + NodeHandlePlan::Inline(range) + }); + } + } + Ok(NodePlan::NibbledBranch { + partial: NibbleSlicePlan::new(partial, partial_padding), + value, + children, + }) + }, + NodeHeader::HashedValueLeaf(nibble_count) | NodeHeader::Leaf(nibble_count) => { + let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; + // check that the padding is valid (if any) + if padding && nibble_ops::pad_left(data[input.offset]) != 0 { + return Err(Error::BadFormat) + } + let partial = input.take( + (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / + nibble_ops::NIBBLE_PER_BYTE, + )?; + let partial_padding = nibble_ops::number_padding(nibble_count); + let value = if contains_hash { + ValuePlan::Node(input.take(H::LENGTH)?) + } else { + let count = >::decode(&mut input)?.0 as usize; + ValuePlan::Inline(input.take(count)?) + }; + + Ok(NodePlan::Leaf { + partial: NibbleSlicePlan::new(partial, partial_padding), + value, + }) + }, + } + } + + fn is_empty_node(data: &[u8]) -> bool { + data == ::empty_node() + } + + fn empty_node() -> &'static [u8] { + &[trie_constants::EMPTY_TRIE] + } + + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + let contains_hash = matches!(&value, Value::Node(..)); + let mut output = if contains_hash { + partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) + } else { + partial_from_iterator_encode(partial, number_nibble, NodeKind::Leaf) + }; + match value { + Value::Inline(value) => { + Compact(value.len() as u32).encode_to(&mut output); + output.extend_from_slice(value); + }, + Value::Node(hash) => { + debug_assert!(hash.len() == H::LENGTH); + output.extend_from_slice(hash); + }, + } + output + } + + fn extension_node( + _partial: impl Iterator, + _nbnibble: usize, + _child: ChildReference<::Out>, + ) -> Vec { + unreachable!("No extension codec.") + } + + fn branch_node( + _children: impl Iterator::Out>>>>, + _maybe_value: Option, + ) -> Vec { + unreachable!("No extension codec.") + } + + fn branch_node_nibbled( + partial: impl Iterator, + number_nibble: usize, + children: impl Iterator::Out>>>>, + value: Option, + ) -> Vec { + let contains_hash = matches!(&value, Some(Value::Node(..))); + let mut output = match (&value, contains_hash) { + (&None, _) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), + (_, false) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), + (_, true) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueBranch), + }; + + let bitmap_index = output.len(); + let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; + (0..BITMAP_LENGTH).for_each(|_| output.push(0)); + match value { + Some(Value::Inline(value)) => { + Compact(value.len() as u32).encode_to(&mut output); + output.extend_from_slice(value); + }, + Some(Value::Node(hash)) => { + debug_assert!(hash.len() == H::LENGTH); + output.extend_from_slice(hash); + }, + None => (), + } + Bitmap::encode( + children.map(|maybe_child| match maybe_child.borrow() { + Some(ChildReference::Hash(h)) => { + h.as_ref().encode_to(&mut output); + true + }, + &Some(ChildReference::Inline(inline_data, len)) => { + inline_data.as_ref()[..len].encode_to(&mut output); + true + }, + None => false, + }), + bitmap.as_mut(), + ); + output[bitmap_index..bitmap_index + BITMAP_LENGTH] + .copy_from_slice(&bitmap[..BITMAP_LENGTH]); + output + } +} + +// utils + +/// Encode and allocate node type header (type and size), and partial value. +/// It uses an iterator over encoded partial bytes as input. +fn partial_from_iterator_encode>( + partial: I, + nibble_count: usize, + node_kind: NodeKind, +) -> Vec { + let mut output = Vec::with_capacity(4 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); + match node_kind { + NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), + NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), + NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), + }; + output.extend(partial); + output +} + +const BITMAP_LENGTH: usize = 2; + +/// Radix 16 trie, bitmap encoding implementation, +/// it contains children mapping information for a branch +/// (children presence only), it encodes into +/// a compact bitmap encoding representation. +pub(crate) struct Bitmap(u16); + +impl Bitmap { + pub fn decode(data: &[u8]) -> Result { + let value = u16::decode(&mut &data[..])?; + if value == 0 { + Err("Bitmap without a child.".into()) + } else { + Ok(Bitmap(value)) + } + } + + pub fn value_at(&self, i: usize) -> bool { + self.0 & (1u16 << i) != 0 + } + + pub fn encode>(has_children: I, dest: &mut [u8]) { + let mut bitmap: u16 = 0; + let mut cursor: u16 = 1; + for v in has_children { + if v { + bitmap |= cursor + } + cursor <<= 1; + } + dest[0] = (bitmap % 256) as u8; + dest[1] = (bitmap / 256) as u8; + } +} + +/// substrate trie layout +pub struct LayoutV0(PhantomData); + +/// substrate trie layout, with external value nodes. +pub struct LayoutV1(PhantomData); + +impl TrieLayout for LayoutV0 +where + H: Hasher + core::fmt::Debug, +{ + const USE_EXTENSION: bool = false; + const ALLOW_EMPTY: bool = true; + const MAX_INLINE_VALUE: Option = None; + + type Hash = H; + type Codec = NodeCodec; +} + +impl TrieConfiguration for LayoutV0 +where + H: Hasher + core::fmt::Debug, +{ + fn trie_root(input: I) -> ::Out + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::trie_root_no_extension::(input, Self::MAX_INLINE_VALUE) + } + + fn trie_root_unhashed(input: I) -> Vec + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::unhashed_trie_no_extension::( + input, + Self::MAX_INLINE_VALUE, + ) + } + + fn encode_index(input: u32) -> Vec { + codec::Encode::encode(&codec::Compact(input)) + } +} + +impl TrieLayout for LayoutV1 +where + H: Hasher + core::fmt::Debug, +{ + const USE_EXTENSION: bool = false; + const ALLOW_EMPTY: bool = true; + const MAX_INLINE_VALUE: Option = Some(TRIE_VALUE_NODE_THRESHOLD); + + type Hash = H; + type Codec = NodeCodec; +} + +impl TrieConfiguration for LayoutV1 +where + H: Hasher + core::fmt::Debug, +{ + fn trie_root(input: I) -> ::Out + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::trie_root_no_extension::(input, Self::MAX_INLINE_VALUE) + } + + fn trie_root_unhashed(input: I) -> Vec + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::unhashed_trie_no_extension::( + input, + Self::MAX_INLINE_VALUE, + ) + } + + fn encode_index(input: u32) -> Vec { + codec::Encode::encode(&codec::Compact(input)) + } +} + +/// A node header +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum NodeHeader { + Null, + // contains wether there is a value and nibble count + Branch(bool, usize), + // contains nibble count + Leaf(usize), + // contains nibble count. + HashedValueBranch(usize), + // contains nibble count. + HashedValueLeaf(usize), +} + +impl NodeHeader { + pub(crate) fn contains_hash_of_value(&self) -> bool { + matches!(self, NodeHeader::HashedValueBranch(_) | NodeHeader::HashedValueLeaf(_)) + } +} + +/// NodeHeader without content +pub(crate) enum NodeKind { + Leaf, + BranchNoValue, + BranchWithValue, + HashedValueLeaf, + HashedValueBranch, +} + +impl Encode for NodeHeader { + fn encode_to(&self, output: &mut T) { + match self { + NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), + NodeHeader::Branch(true, nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), + NodeHeader::Branch(false, nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::BRANCH_WITHOUT_MASK, + 2, + output, + ), + NodeHeader::Leaf(nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), + NodeHeader::HashedValueBranch(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_BRANCH_WITH_MASK, + 4, + output, + ), + NodeHeader::HashedValueLeaf(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, + 3, + output, + ), + } + } +} + +impl codec::EncodeLike for NodeHeader {} + +impl Decode for NodeHeader { + fn decode(input: &mut I) -> Result { + let i = input.read_byte()?; + if i == trie_constants::EMPTY_TRIE { + return Ok(NodeHeader::Null) + } + match i & (0b11 << 6) { + trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITH_MASK => + Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => + Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::EMPTY_TRIE => { + if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { + Ok(NodeHeader::HashedValueLeaf(decode_size(i, input, 3)?)) + } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { + Ok(NodeHeader::HashedValueBranch(decode_size(i, input, 4)?)) + } else { + // do not allow any special encoding + Err("Unallowed encoding".into()) + } + }, + _ => unreachable!(), + } + } +} + +/// Returns an iterator over encoded bytes for node header and size. +/// Size encoding allows unlimited, length inefficient, representation, but +/// is bounded to 16 bit maximum value to avoid possible DOS. +pub(crate) fn size_and_prefix_iterator( + size: usize, + prefix: u8, + prefix_mask: usize, +) -> impl Iterator { + let max_value = 255u8 >> prefix_mask; + let l1 = core::cmp::min((max_value as usize).saturating_sub(1), size); + let (first_byte, mut rem) = if size == l1 { + (once(prefix + l1 as u8), 0) + } else { + (once(prefix + max_value as u8), size - l1) + }; + let next_bytes = move || { + if rem > 0 { + if rem < 256 { + let result = rem - 1; + rem = 0; + Some(result as u8) + } else { + rem = rem.saturating_sub(255); + Some(255) + } + } else { + None + } + }; + first_byte.chain(core::iter::from_fn(next_bytes)) +} + +/// Encodes size and prefix to a stream output. +fn encode_size_and_prefix(size: usize, prefix: u8, prefix_mask: usize, out: &mut W) +where + W: Output + ?Sized, +{ + for b in size_and_prefix_iterator(size, prefix, prefix_mask) { + out.push_byte(b) + } +} + +/// Decode size only from stream input and header byte. +fn decode_size( + first: u8, + input: &mut impl Input, + prefix_mask: usize, +) -> Result { + let max_value = 255u8 >> prefix_mask; + let mut result = (first & max_value) as usize; + if result < max_value as usize { + return Ok(result) + } + result -= 1; + loop { + let n = input.read_byte()? as usize; + if n < 255 { + return Ok(result + n + 1) + } + result += 255; + } +} + +/// Error type used for trie related errors. +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Error { + BadFormat, + Decode(codec::Error), + InvalidRecording(Vec, bool), + TrieError(Box>), +} + +impl core::fmt::Display for Error { + fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { + fmt.write_str("Error") + } +} + +impl std::error::Error for Error where H: core::fmt::Debug {} + +impl From for Error { + fn from(x: codec::Error) -> Self { + Error::Decode(x) + } +} + +impl From>> for Error { + fn from(x: Box>) -> Self { + Error::TrieError(x) + } +} From 8910260d85d14fd5a1744020177830d722b4d632 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 17 Mar 2023 09:33:42 +0000 Subject: [PATCH 3/6] Fix `TrieDBRawIterator::prefix_then_seek` --- trie-db/fuzz/Cargo.toml | 6 + trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs | 8 ++ trie-db/fuzz/src/lib.rs | 60 +++++++++ trie-db/src/iterator.rs | 79 +++++++---- trie-db/test/Cargo.toml | 3 + trie-db/test/src/triedb.rs | 123 +++++++++++++++++- 6 files changed, 251 insertions(+), 28 deletions(-) create mode 100644 trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index 889212f0..84a03257 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -12,6 +12,8 @@ cargo-fuzz = true hash-db = { path = "../../hash-db", version = "0.16.0" } memory-db = { path = "../../memory-db", version = "0.32.0" } reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } +arbitrary = { version = "1.3.0", features = ["derive"] } +array-bytes = "6.0.0" [dependencies.trie-db] path = ".." @@ -62,3 +64,7 @@ path = "fuzz_targets/trie_codec_proof.rs" [[bin]] name = "trie_proof_invalid" path = "fuzz_targets/trie_proof_invalid.rs" + +[[bin]] +name = "prefix_seek_iter" +path = "fuzz_targets/prefix_seek_iter.rs" diff --git a/trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs b/trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs new file mode 100644 index 00000000..23e1f734 --- /dev/null +++ b/trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use trie_db_fuzz::{fuzz_prefix_seek_iter, PrefixSeekTestInput}; + +fuzz_target!(|data: PrefixSeekTestInput| { + fuzz_prefix_seek_iter::>(data); +}); diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 1003c783..f9de8c07 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use arbitrary::Arbitrary; use hash_db::Hasher; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ @@ -258,6 +259,65 @@ pub fn fuzz_prefix_iter(input: &[u8]) { assert_eq!(error, 0); } +#[derive(Debug, Arbitrary)] +pub struct PrefixSeekTestInput { + keys: Vec>, + prefix_key: Vec, + seek_key: Vec, +} + +fn printable_keys>(iter: impl IntoIterator) -> String { + iter.into_iter() + .map(|key| format!("\"{}\"", array_bytes::bytes2hex("", key))) + .collect::>() + .join(", ") +} + +pub fn fuzz_prefix_seek_iter(mut input: PrefixSeekTestInput) { + type PrefixedMemoryDB = + MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; + + input.keys.retain_mut(|key| !key.is_empty()); + + input.keys.sort_unstable(); + input.keys.dedup(); + + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (index, key) in input.keys.iter().enumerate() { + t.insert(&key, &[index as u8]).unwrap(); + } + } + + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + let iter = + trie_db::TrieDBIterator::new_prefixed_then_seek(&trie, &input.prefix_key, &input.seek_key) + .unwrap(); + let output_keys: Vec<_> = iter.map(|item| item.unwrap().0).collect(); + + let input_keys = input.keys; + let seek_key = input.seek_key; + let prefix_key = input.prefix_key; + let expected_keys: Vec<_> = input_keys + .iter() + .filter(|key| key.starts_with(&prefix_key) && **key >= seek_key) + .cloned() + .collect(); + + if output_keys != expected_keys { + panic!( + "Test failed!\nresult = [{result}]\nexpected = [{expected}]\nprefix_key = \"{prefix_key}\"\nseek_key = \"{seek_key}\"\nkeys = [{input_keys}]", + result = printable_keys(output_keys), + expected = printable_keys(expected_keys), + prefix_key = array_bytes::bytes2hex("", prefix_key), + seek_key = array_bytes::bytes2hex("", seek_key), + input_keys = printable_keys(input_keys) + ); + } +} + pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) { let mut data = fuzz_to_data(input); // Split data into 3 parts: diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 9b7d0710..ca382b70 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -288,35 +288,60 @@ impl TrieDBRawIterator { prefix: &[u8], seek: &[u8], ) -> Result<(), TrieHash, CError> { - if seek.starts_with(prefix) { - self.seek(db, seek)?; - let prefix_len = prefix.len() * crate::nibble::nibble_ops::NIBBLE_PER_BYTE; - let mut len = 0; - // look first prefix in trail - for i in 0..self.trail.len() { - match self.trail[i].node.node_plan() { - NodePlan::Empty => {}, - NodePlan::Branch { .. } => { - len += 1; - }, - NodePlan::Leaf { partial, .. } => { - len += partial.len(); - }, - NodePlan::Extension { partial, .. } => { - len += partial.len(); - }, - NodePlan::NibbledBranch { partial, .. } => { - len += 1; - len += partial.len(); - }, - } - if len > prefix_len { - self.trail = self.trail.split_off(i); - return Ok(()) - } + if prefix.is_empty() { + // There's no prefix, so just seek. + return self.seek(db, seek).map(|_| ()) + } + + if seek.is_empty() || seek <= prefix { + // Either we're not supposed to seek anywhere, + // or we're supposed to seek *before* the prefix, + // so just directly go to the prefix. + return self.prefix(db, prefix) + } + + if !seek.starts_with(prefix) { + // We're supposed to seek *after* the prefix, + // so just return an empty iterator. + self.trail.clear(); + return Ok(()) + } + + if !self.seek(db, prefix)? { + // The database doesn't have a key with such a prefix. + self.trail.clear(); + return Ok(()) + } + + // Now seek forward again. + self.seek(db, seek)?; + + let prefix_len = prefix.len() * crate::nibble::nibble_ops::NIBBLE_PER_BYTE; + let mut len = 0; + // look first prefix in trail + for i in 0..self.trail.len() { + match self.trail[i].node.node_plan() { + NodePlan::Empty => {}, + NodePlan::Branch { .. } => { + len += 1; + }, + NodePlan::Leaf { partial, .. } => { + len += partial.len(); + }, + NodePlan::Extension { partial, .. } => { + len += partial.len(); + }, + NodePlan::NibbledBranch { partial, .. } => { + len += 1; + len += partial.len(); + }, + } + if len > prefix_len { + self.trail = self.trail.split_off(i); + return Ok(()) } } - // default to empty iter + self.trail.clear(); Ok(()) } diff --git a/trie-db/test/Cargo.toml b/trie-db/test/Cargo.toml index 2da9f07b..547df945 100644 --- a/trie-db/test/Cargo.toml +++ b/trie-db/test/Cargo.toml @@ -22,3 +22,6 @@ hex-literal = "0.3" criterion = "0.4.0" env_logger = { version = "0.10", default-features = false } log = "0.4" + +[dev-dependencies] +array-bytes = "6.0.0" diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index e4420ae3..9825ab50 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -17,7 +17,9 @@ use std::ops::Deref; use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use hex_literal::hex; use memory_db::{HashKey, MemoryDB, PrefixedKey}; -use reference_trie::{test_layouts, HashedValueNoExtThreshold, TestTrieCache}; +use reference_trie::{ + test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, TestTrieCache, +}; use trie_db::{ encode_compact, CachedValue, DBValue, Lookup, NibbleSlice, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut, @@ -169,6 +171,125 @@ fn iterator_seek_internal() { assert_eq!(&vals[5..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } +fn trie_from_hex_keys(keys: &[&str], callback: impl FnOnce(&mut trie_db::TrieDB)) +where + T: TrieLayout, +{ + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (index, key) in keys.iter().enumerate() { + t.insert(&array_bytes::hex2bytes(key).unwrap(), &[index as u8]).unwrap(); + } + } + + let mut t = TrieDBBuilder::::new(&memdb, &root).build(); + callback(&mut t); +} + +fn test_prefixed_then_seek( + keys: &[&str], + prefix_key: &str, + seek_key: &str, + expected: &[&str], +) { + let prefix_key = array_bytes::hex2bytes(prefix_key).unwrap(); + let seek_key = array_bytes::hex2bytes(seek_key).unwrap(); + + trie_from_hex_keys::(keys, |trie| { + let iter = + trie_db::TrieDBIterator::new_prefixed_then_seek(&trie, &prefix_key, &seek_key).unwrap(); + let output: Vec<_> = iter.map(|x| array_bytes::bytes2hex("", x.unwrap().0)).collect(); + assert_eq!(output, expected); + }); +} + +// This test reproduces an actual real-world issue: https://github.com/polkadot-js/apps/issues/9103 +test_layouts_substrate!(iterator_prefixed_then_seek_real_world); +fn iterator_prefixed_then_seek_real_world() { + let keys = &[ + "6cf4040bbce30824850f1a4823d8c65faeefaa25a5bae16a431719647c1d99da", + "6cf4040bbce30824850f1a4823d8c65ff536928ca5ba50039bc2766a48ddbbab", + "70f943199f1a2dde80afdaf3f447db834e7b9012096b41c4eb3aaf947f6ea429", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d007fc7effcb0c044a0c41fd8a77eb55d2133058a86d1f4d6f8e45612cd271eefd77f91caeaacfe011b8f41540e0a793b0fd51b245dae19382b45386570f2b545fab75e3277910f7324b55f47c29f9965e8298371404e50ac", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d0179c23cd593c770fde9fc7aa8f84b3e401e654b8986c67728844da0080ec9ee222b41a85708a471a511548302870b53f40813d8354b6d2969e1b7ca9e083ecf96f9647e004ecb41c7f26f0110f778bdb3d9da31bef323d9", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d024de296f88310247001277477f4ace4d0aa5685ea2928d518a807956e4806a656520d6520b8ac259f684aa0d91961d76f697716f04e6c997338d03560ab7d703829fe7b9d0e6d7eff8d8412fc428364c2f474a67b36586d", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d13dc5d83f2361c14d05933eb3182a92ac14665718569703baf1da25c7d571843b6489f03d8549c87bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d1786d20bbb4b91eb1f5765432d750bd0111a0807c8d04f05110ffaf73f4fa7b360422c13bc97efc3a2324d9fa8f954b424c0bcfce7236a2e8107dd31c2042a9860a964f8472fda49749dec3f146e81470b55aa0f3930d854", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d18c246484ec5335a40903e7cd05771be7c0b8459333f1ae2925c3669fc3e5accd0f38c4711a15544bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d1aca749033252ce75245528397430d14cb8e8c09248d81ee5de00b6ae93ee880b6d19a595e6dc106bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d1d6bceb91bc07973e7b3296f83af9f1c4300ce9198cc3b44c54dafddb58f4a43aee44a9bef1a2e9dbfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d203383772f45721232139e1a8863b0f2f8d480bdc15bcc1f2033cf467e137059558da743838f6b58bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d2197cc5c3eb3a6a67538e0dc3eaaf8c820d71310d377499c4a5d276381789e0a234475e69cddf709d207458083d6146d3a36fce7f1fe05b232702bf154096e5e3a8c378bdc237d7a27909acd663563917f0f70bb0e8e61a3", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d4f19c117f2ea36100f753c4885aa8d63b4d65a0dc32106f829f89eeabd52c37105c9bdb75f752469729fa3f0e7d907c1d949192c8e264a1a510c32abe3a05ed50be2262d5bfb981673ec80a07fd2ce28c7f27cd0043a788c", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d547d5aaa651bafa63d077560dfe823ac75665ebf1dcfd96a06e45499f03dda31282977706918d4821b8f41540e0a793b0fd51b245dae19382b45386570f2b545fab75e3277910f7324b55f47c29f9965e8298371404e50ac", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d6037207d54d69a082ea225ab4a412e4b87d6f5612053b07c405cf05ea25e482a4908c0713be2998abfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d63d0920de0c7315ebaed1d639d926961d28af89461c31eca890441e449147d23bb7c9d4fc42d7c16bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d7912c66be82a5972e5bc11c8d10551a296ba9aaff8ca6ab22a8cd1987974b87a97121c871f786d2e17e0a629acf01c38947f170b7e02a9ebb4ee60f83779acb99b71114c01a4f0a60694611a1502c399c77214ffa26e955b", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d7aa00f217f3a374a2f1ca0f388719f84099e8157a8a83c5ccf54eae1617f93933fa976baa629e6febfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d9e1c3c8ab41943cf377b1aa724d7f518a3cfc96a732bdc4658155d09ed2bfc31b5ccbc6d8646b59f1b8f41540e0a793b0fd51b245dae19382b45386570f2b545fab75e3277910f7324b55f47c29f9965e8298371404e50ac", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d9fb8d6d95d5214a3305a4fa07e344eb99fad4be3565d646c8ac5af85514d9c96702c9c207be234958dbdb9185f467d2be3b84e8b2f529f7ec3844b378a889afd6bd31a9b5ed22ffee2019ad82c6692f1736dd41c8bb85726", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8d9fb8d6d95d5214a3305a4fa07e344eb99fad4be3565d646c8ac5af85514d9c96702c9c207be23495ec1caa509591a36a8403684384ce40838c9bd7fc49d933a10d3b26e979273e2f17ebf0bf41cd90e4287e126a59d5a243", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8da7fc066aae2ffe03b36e9a72f9a39cb2befac7e47f320309f31f1c1676288d9596045807304b3d79bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8daf3c377b0fddf7c7ad6d390fab0ab45ac16c21645be880af5cab2fbbeb04820401a4c9f766c17bef9fc14a2e16ade86fe26ee81d4497dc6aab81cc5f5bb0458d6149a763ecb09aefec06950dd61db1ba025401d2a04e3b9d", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8daf3c377b0fddf7c7ad6d390fab0ab45ac16c21645be880af5cab2fbbeb04820401a4c9f766c17befbfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8db60505ba8b77ef03ed805436d3242f26dc828084b12aaf4bcb96af468816a182b5360149398aad6b1dafe949b0918138ceef924f6393d1818a04842301294604972da17b24b31b155e4409a01273733b8d21a156c2e7eb71", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8dbd27136a6e028656073cc840bfabb48fe935880c4c4c990ee98458b2fed308e9765f7f7f717dd3b2862fa5361d3b55afa6040e582687403c852b2d065b24f253276cc581226991f8e1818a78fc64c39da7f0b383c6726e0f", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8dca40d91320edd326500f9e8b5a0b23a8bdf21549f98f0e014f66b6a18bdd78e337a6c05d670c80c88a55d4c7bb6fbae546e2d03ac9ab16e85fe11dad6adfd6a20618905477b831d7d48ca32d0bfd2bdc8dbeba26ffe2c710", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8dd27478512243ed62c1c1f7066021798a464d4cf9099546d5d9907b3369f1b9d7a5aa5d60ca845619bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8de6da5659cbbe1489abbe99c4d3a474f4d1e78edb55a9be68d8f52c6fe730388a298e6f6325db3da7bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8de6da5659cbbe1489abbe99c4d3a474f4d1e78edb55a9be68d8f52c6fe730388a298e6f6325db3da7e94ca3e8c297d82f71e232a2892992d1f6480475fb797ce64e58f773d8fafd9fbcee4bdf4b14f2a71b6d3a428cf9f24b", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8decdd1760c61ff7234f2876dbe817af803170233320d778b92043b2359e3de6d16c9e5359f6302da31c84d6f551ad2a831263ef956f0cdb3b4810cefcb2d0b57bcce7b82007016ae4fe752c31d1a01b589a7966cea03ec65c", + "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8df9981ee6b69eb7af2153af34f39ffc06e2daa5272c99798c8849091284dc8905f2a76b65754c2089bfa5709836ba729443c319659e83ad5ee133e6f11af51d883e56216e9e1bbb1e2920c7c6120cbb55cd469b1f95b61601", + "7474449cca95dc5d0c00e71735a6d17d4e7b9012096b41c4eb3aaf947f6ea429", + "89d139e01a5eb2256f222e5fc5dbe6b33c9c1284130706f5aea0c8b3d4c54d89", + "89d139e01a5eb2256f222e5fc5dbe6b36254e9d55588784fa2a62b726696e2b1" + ]; + + let target_key = "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8da7dad55cf08ffe8194efa962146801b0503092b1ed6a3fa6aee9107334aefd7965bbe568c3d24c6d"; + test_prefixed_then_seek::(keys, target_key, target_key, &[]); +} + +// This is the real-word test, but simplified. +test_layouts_substrate!(iterator_prefixed_then_seek_simple); +fn iterator_prefixed_then_seek_simple() { + test_prefixed_then_seek::(&["0100"], "00", "00", &[]); +} + +// These are just tests that the fuzzer barfed out while working on the fix for the real-world +// issue. +test_layouts_substrate!(iterator_prefixed_then_seek_testcase_1); +fn iterator_prefixed_then_seek_testcase_1() { + test_prefixed_then_seek::(&["00"], "00", "", &["00"]) +} + +test_layouts_substrate!(iterator_prefixed_then_seek_testcase_2); +fn iterator_prefixed_then_seek_testcase_2() { + test_prefixed_then_seek::(&["00", "0003"], "00", "", &["00", "0003"]) +} + +test_layouts_substrate!(iterator_prefixed_then_seek_testcase_3); +fn iterator_prefixed_then_seek_testcase_3() { + test_prefixed_then_seek::(&["20"], "20", "0700", &["20"]) +} + +test_layouts_substrate!(iterator_prefixed_then_seek_testcase_4); +fn iterator_prefixed_then_seek_testcase_4() { + let keys = &["1701", "ffffffffffffffffffffffdfffffffffffffffffffffffffffffffffffffffff"]; + test_prefixed_then_seek::( + keys, + "1701", + "ffff27272727274949494949ce494949494949494949491768687b737373732b", + &[], + ) +} + +test_layouts_substrate!(iterator_prefixed_then_seek_testcase_5); +fn iterator_prefixed_then_seek_testcase_5() { + test_prefixed_then_seek::(&["20"], "20", "20", &["20"]) +} + test_layouts!(get_length_with_extension, get_length_with_extension_internal); fn get_length_with_extension_internal() { let mut memdb = PrefixedMemoryDB::::default(); From b1d23de51640ea5427c62a26d8fff7d32d06ec60 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 17 Mar 2023 09:47:39 +0000 Subject: [PATCH 4/6] Bump `trie-db` to 0.27.1 and `reference-trie` to 0.29.1 --- test-support/reference-trie/Cargo.toml | 2 +- trie-db/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 93404dd8..90fe577a 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "reference-trie" -version = "0.29.0" +version = "0.29.1" authors = ["Parity Technologies "] description = "Simple reference trie format" repository = "https://github.com/paritytech/trie/" diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index 78f246fc..f8e6b00c 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "trie-db" -version = "0.27.0" +version = "0.27.1" authors = ["Parity Technologies "] description = "Merkle-Patricia Trie generic over key hasher and node encoding" repository = "https://github.com/paritytech/trie" From 8250bde4e9ce02cfd091bf77904dabcace3252c9 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 17 Mar 2023 09:47:59 +0000 Subject: [PATCH 5/6] Update changelogs --- test-support/reference-trie/CHANGELOG.md | 2 ++ trie-db/CHANGELOG.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/test-support/reference-trie/CHANGELOG.md b/test-support/reference-trie/CHANGELOG.md index c0712c17..aeafd12a 100644 --- a/test-support/reference-trie/CHANGELOG.md +++ b/test-support/reference-trie/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog]. ## [Unreleased] +## [0.29.1] - 2023-03-17 +- Add substrate trie layouts ## [0.29.0] - 2023-03-14 - Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) diff --git a/trie-db/CHANGELOG.md b/trie-db/CHANGELOG.md index d38f648f..90766d03 100644 --- a/trie-db/CHANGELOG.md +++ b/trie-db/CHANGELOG.md @@ -4,6 +4,9 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ +## [0.27.1] - 2023-03-17 +- Fix `TrieDBRawIterator::prefix_then_seek` + ## [0.27.0] - 2023-03-14 - Fix compact proof to skip including value node hashes [#187](https://github.com/paritytech/trie/pull/187) - Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) From d5e7703fed59c5b2feaff9a6b9b2d1a96d2cf8ce Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 17 Mar 2023 11:17:51 +0000 Subject: [PATCH 6/6] Add PR URL to the changelogs --- test-support/reference-trie/CHANGELOG.md | 2 +- trie-db/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-support/reference-trie/CHANGELOG.md b/test-support/reference-trie/CHANGELOG.md index aeafd12a..d2c05f22 100644 --- a/test-support/reference-trie/CHANGELOG.md +++ b/test-support/reference-trie/CHANGELOG.md @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog]. ## [Unreleased] ## [0.29.1] - 2023-03-17 -- Add substrate trie layouts +- Add substrate trie layouts [#190](https://github.com/paritytech/trie/pull/190) ## [0.29.0] - 2023-03-14 - Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) diff --git a/trie-db/CHANGELOG.md b/trie-db/CHANGELOG.md index 90766d03..ac450d40 100644 --- a/trie-db/CHANGELOG.md +++ b/trie-db/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [0.27.1] - 2023-03-17 -- Fix `TrieDBRawIterator::prefix_then_seek` +- Fix `TrieDBRawIterator::prefix_then_seek` [#190](https://github.com/paritytech/trie/pull/190) ## [0.27.0] - 2023-03-14 - Fix compact proof to skip including value node hashes [#187](https://github.com/paritytech/trie/pull/187)