From 401bc2e6fa68ec9d500f4aaf19d3d35d4ffab975 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 26 Feb 2020 10:02:01 +0100 Subject: [PATCH 01/73] complex proof (binary tree hash). --- Cargo.toml | 1 + hash-db/src/lib.rs | 61 +- memory-db/Cargo.toml | 2 + memory-db/src/lib.rs | 47 +- ordered-trie/Cargo.toml | 23 + ordered-trie/src/lib.rs | 1558 ++++++++++++++++++++++++ test-support/keccak-hasher/Cargo.toml | 1 + test-support/keccak-hasher/src/lib.rs | 36 + test-support/reference-trie/Cargo.toml | 1 + test-support/reference-trie/src/lib.rs | 375 ++++-- trie-db/Cargo.toml | 2 + trie-db/benches/bench.rs | 4 +- trie-db/fuzz/src/lib.rs | 2 +- trie-db/src/fatdbmut.rs | 3 +- trie-db/src/iter_build.rs | 247 +++- trie-db/src/lib.rs | 17 +- trie-db/src/node.rs | 18 + trie-db/src/node_codec.rs | 221 +++- trie-db/src/proof/generate.rs | 148 ++- trie-db/src/proof/mod.rs | 6 +- trie-db/src/proof/verify.rs | 178 ++- trie-db/src/recorder.rs | 2 +- trie-db/src/sectriedbmut.rs | 3 +- trie-db/src/trie_codec.rs | 344 +++++- trie-db/src/triedbmut.rs | 101 +- 25 files changed, 3116 insertions(+), 285 deletions(-) create mode 100644 ordered-trie/Cargo.toml create mode 100644 ordered-trie/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index a974bf7e..35cd4cbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "hash-db", "memory-db", "hash256-std-hasher", + "ordered-trie", "test-support/keccak-hasher", "test-support/reference-trie", "test-support/trie-standardmap", diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 249ae4eb..8272d23e 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -32,7 +32,6 @@ pub trait MaybeDebug {} #[cfg(not(feature = "std"))] impl MaybeDebug for T {} - /// A trie node prefix, it is the nibble path from the trie root /// to the trie node. /// For a node containing no partial key value it is the full key. @@ -130,6 +129,18 @@ pub trait HashDB: Send + Sync + AsHashDB { fn remove(&mut self, key: &H::Out, prefix: Prefix); } +/// Describes how to hash a node given its layout +pub struct ComplexLayout { + pub nb_children: usize, + // can be calculated from decoded node from the + // children bitmap and the children range + pub children: I, // impl iterator < (is_defined, range) > + pub nb_additional_hashes: usize, + // TODO switch to iter?? + pub additional_hashes: I2, +} + + /// Trait for immutable reference of HashDB. pub trait HashDBRef { /// Look up a given hash into the bytes that hash to it, returning None if the @@ -185,3 +196,51 @@ impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { fn as_plain_db(&self) -> &dyn PlainDB { &**self } fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn PlainDB + 'b) { &mut **self } } + +/* + +TODO this is rather shit at this point not delete yet just in case + +/// Fix hash implementation, it needs to have +/// same output length as input for rounds of hashing. +/// TODO consider moving in its own crate. +pub trait FixHash { + type Hasher: Hasher; + /// if true, then when processing two leaf we do a finalize round. + /// TODO I think only one may be insecure but do not remember + /// the rational, also this should be an associated constant + /// (TODO group types parameters) + /// + /// Tells if state/iv can be initiated from first element of pair. + /// + /// TODO could it be skipped in any circumstance or the other way + /// arount, todo check blake3 permutation + /// + /// Need first hashed implies that we cannot use back a state by + /// calling a second hash. + /// TODO write a test case for that !!! + /// TODO rename to need finalize?? + /// TODO for keccack we could only hash(hash1 xor hash2)? + const NEED_FIRST_HASHED: bool; + /// Value of empty hash at some given depth + /// TODO test case it + const EMPTY_HASHES: &'static [&'static [u8]]; + + // The fix hash type of the `Hasher`. -> is Hasher::Out +// type Out: AsRef<[u8]> + AsMut<[u8]> + Default + hash_db::MaybeDebug + PartialEq + Eq +// + Send + Sync + Clone + Copy; + + // The length in bytes of the `Out` output. -> is Hasher::Length +// const LENGTH: usize; + + /// Compute the hash + fn new(first: ::Out) -> Self; + /// Compute the hash + fn hash(&mut self, second: &::Out); + /// Access current state (if NEED_FIRST_HASHED is false). + fn current_state(&self) -> &::Out; + /// Extract hash (if NEED_FIRST_HASHED is true). + fn finalize(self) -> ::Out; +} +*/ + diff --git a/memory-db/Cargo.toml b/memory-db/Cargo.toml index f20f6e4a..59793501 100644 --- a/memory-db/Cargo.toml +++ b/memory-db/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" heapsize = { version = "0.4", optional = true } parity-util-mem = { version = "0.3", default-features = false } hash-db = { path = "../hash-db", default-features = false, version = "0.15.2"} +ordered-trie = { path = "../ordered-trie", default-features = false, version = "0.19.2"} # TODO could probably remove (work in hashdb) hashbrown = { version = "0.6.3", default-features = false, features = [ "ahash" ] } # There's a compilation error with ahash-0.2.17, which is permitted by the 0.2.11 constraint in hashbrown. ahash = "0.2.18" @@ -23,6 +24,7 @@ criterion = "0.2.8" default = ["std"] std = [ "hash-db/std", + "ordered-trie/std", "parity-util-mem/std", ] deprecated = [ "heapsize" ] diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 67e67246..a08ab0f8 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -19,8 +19,9 @@ #[cfg(not(feature = "std"))] extern crate alloc; +use ordered_trie::{HashOnly, HashDBComplex, HasherComplex, BinaryHasher}; use hash_db::{HashDB, HashDBRef, PlainDB, PlainDBRef, Hasher as KeyHasher, - AsHashDB, AsPlainDB, Prefix}; + AsHashDB, AsPlainDB, Prefix, ComplexLayout}; use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; #[cfg(feature = "deprecated")] #[cfg(feature = "std")] @@ -34,6 +35,7 @@ use std::{ marker::PhantomData, cmp::Eq, borrow::Borrow, + ops::Range, }; #[cfg(not(feature = "std"))] @@ -49,6 +51,7 @@ use core::{ marker::PhantomData, cmp::Eq, borrow::Borrow, + ops::Range, }; #[cfg(not(feature = "std"))] @@ -600,6 +603,48 @@ where } } +impl HashDBComplex for MemoryDB +where + H: HasherComplex, + T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: Send + Sync + KeyFunction, +{ + fn insert_complex< + I: Iterator>, + I2: Iterator, + > ( + &mut self, + prefix: Prefix, + value: &[u8], + no_child_value: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> H::Out { + if T::from(value) == self.null_node_data { + return self.hashed_null_node.clone(); + } + + let key = if let Some(key) = H::hash_complex( + no_child_value, + nb_children, + children, + additional_hashes, + proof, + ) { + key + } else { + // invalid proof TODO do dedicated method that can return + // error?? + return self.hashed_null_node.clone(); + }; + + HashDB::emplace(self, key, prefix, value.into()); + key + } +} + impl HashDBRef for MemoryDB where H: KeyHasher, diff --git a/ordered-trie/Cargo.toml b/ordered-trie/Cargo.toml new file mode 100644 index 00000000..74b18f94 --- /dev/null +++ b/ordered-trie/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "ordered-trie" +version = "0.19.2" +authors = ["Parity Technologies "] +description = "Binary tree generic for ordered values (stack or fifo)" +repository = "https://github.com/paritytech/trie" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +hash-db = { path = "../hash-db", default-features = false, version = "0.15.2"} +smallvec = "1.0.0" + +[dev-dependencies] +keccak-hasher = { path = "../test-support/keccak-hasher", version = "0.15.2"} +#criterion = "0.2.8" + + +[features] +default = ["std"] +std = [ + "hash-db/std", +] diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs new file mode 100644 index 00000000..36792272 --- /dev/null +++ b/ordered-trie/src/lib.rs @@ -0,0 +1,1558 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg_attr(not(feature = "std"), no_std)] + +//! This crate contains implementation of trie/tree based on ordered sequential key only. +//! +//! Targetted use case is a stack or a fifo. + +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(feature = "std")] +mod rstd { + pub use std::{borrow, boxed, cmp, convert, fmt, hash, iter, marker, mem, ops, rc, result, vec}; + pub use std::collections::VecDeque; + pub use std::collections::BTreeMap; + pub use std::error::Error; +} + +#[cfg(not(feature = "std"))] +mod rstd { + pub use core::{borrow, convert, cmp, iter, fmt, hash, marker, mem, ops, result}; + pub use alloc::{boxed, rc, vec}; + pub use alloc::collections::VecDeque; + pub trait Error {} + impl Error for T {} +} + +#[cfg(feature = "std")] +use self::rstd::{fmt, Error}; + +use hash_db::{MaybeDebug, AsHashDB, Prefix, HashDB}; +use self::rstd::{boxed::Box, vec::Vec}; + + +use hash_db::{HashDBRef, Hasher}; +use crate::rstd::marker::PhantomData; + + +pub type DBValue = Vec; + +pub mod key { + /// Base type for key, TODO + /// implementing on usize for now, + /// then this will probably look like + /// substrate simple arithmetic trait. + /// -> need an implementation for [u8] + /// (unbounded key length) + pub trait OrderedKey: Ord {} + + impl OrderedKey for T {} +} + + + +pub mod meta { + use hash_db::Hasher; + + /// Codec for the meta needed to get + /// information for the tree. + pub trait MetaCodec { + /// If false we do not associate + /// meta to the trie, their value + /// should be trusted from an external + /// source (eg inclusion in the header + /// of a block like root), in this case + /// the implementation of this trait should + /// never be use. Then trie root is the binary_root. + /// If true the code will be use to produce + /// a root with those information and the + /// root will be hash(meta ++ binary_root), + /// so at minimal an additional round of + /// hashing. Binary root is included in meta. + /// + /// A node contaning this `meta ++ binary_root` + /// is using a prefix of length 0. + const ATTACH_TO_ROOT: bool; + + /// The buffer for this codec, this allows + /// to use fix length buffer. + type Buff: AsMut<[u8]> + AsRef<[u8]>; + /// The hash to use if we need + /// to associate meta. + type Hash: Hasher; + /// The actual meta to use. + type Meta; + /// Decode + fn decode(input: &[u8]) -> Self::Meta; + /// Encode TODO use write and stream trait? + fn encode(meta: &Self::Meta) -> Vec; + } + + /// Direct hasher as meta indicates there + /// is no meta so we can use the root directly. + impl MetaCodec for H { + const ATTACH_TO_ROOT: bool = false; + type Buff = [u8;0]; + type Hash = Self; + type Meta = (); + fn decode(_input: &[u8]) -> Self::Meta { () } + fn encode(_meta: &Self::Meta) -> Vec { Vec::new() } + } + +} + +#[derive(PartialEq, Eq, Debug)] +/// A binary trie with guaranties +/// of content being in a fix range +/// of sequential values. +pub struct SequenceBinaryTree { + // Metadata (needs to be validated) + + /// global offset for index. + offset: K, + /// Nb deleted values at start. + start: K, + start_depth: usize, + /// Nb deleted values at end. + end: K, + end_depth: usize, + + // depth of the full tree (maximum depth) + depth: usize, + // memmoïze 2^depth + length: K, + + _ph: PhantomData, +} + +pub struct SequenceBinaryTreeDB<'a, K, H: Hasher> { + tree: &'a SequenceBinaryTree, + db: &'a dyn HashDBRef, + root: &'a H::Out, +} + +pub struct SequenceBinaryTreeInMem<'a, K, NK: Ord, H: Hasher> { + tree: &'a SequenceBinaryTree, + db: &'a crate::rstd::BTreeMap, +} + +impl Default for SequenceBinaryTree { + fn default() -> Self { + SequenceBinaryTree { + offset: 0, + start: 0, + start_depth: 0, + end: 0, + end_depth: 0, + depth: 0, + length: 0, + _ph: PhantomData, + } + } +} + +fn depth(nb: usize) -> usize { + (0usize.leading_zeros() - nb.leading_zeros()) as usize +/* if nb == 0 { + return 0; + } + + ((0usize.leading_zeros() - (nb - 1).leading_zeros()) as usize) + 1*/ +} + +#[test] +fn test_depth() { +/* + (0, 0), + (1, 1), + (2, 2), + (3, 2), + (4, 3), + (5, 3), + (7, 3), + (8, 4), +*/ + assert_eq!(depth(0), 0); + assert_eq!(depth(1), 1); + assert_eq!(depth(2), 2); + assert_eq!(depth(3), 2); + assert_eq!(depth(4), 3); + assert_eq!(depth(7), 3); + assert_eq!(depth(8), 4); + assert_eq!(depth(9), 4); + assert_eq!(depth(u16::max_value() as usize - 1), 16); + assert_eq!(depth(u16::max_value() as usize), 16); +} + +fn right_at(value: usize, index: usize) -> bool { + value & (1 << index) != 0 +} + +impl SequenceBinaryTree { + pub fn new(offset: usize, start: usize, number: usize) -> Self { + let len = start + number; + if len == 0 { + SequenceBinaryTree { + offset, + start, + start_depth: 0, + end: 0, + end_depth: 0, + depth: 0, + length: 0, + _ph: PhantomData, + } + } else { + let length = len.next_power_of_two(); + let end = length - start - number; + let start_depth = depth(start); + let end_depth = depth(end); + let depth = depth(length - 1); + SequenceBinaryTree { + offset, + start, + start_depth, + end, + end_depth, + depth, + length, + _ph: PhantomData, + } + } + } + + // TODO consider storing that + fn nb_elements(&self) -> usize { + self.length - self.start - self.end + } + + fn push(&mut self, mut nb: usize) { + if nb == 0 { + return; + } + if self.length == 0 { + *self = Self::new(self.offset, self.start, nb); + return; + } + while nb > self.end { + nb -= self.end; + self.depth += 1; + if self.length == 0 { + self.length += 1; + self.end = 1; + } else { + self.end = self.length; + self.length *= 2; + } + + } + self.end -= nb; + self.end_depth = depth(self.end); + } + + fn depth_index(&self, index: usize) -> usize { + let tmp = (!0usize << (self.depth - self.end_depth)) | (index >> self.end_depth); + if !tmp == 0 { + let mut nb_skip = 0; + for i in 0..self.end_depth { + let ix = self.end_depth - i - 1; // - 1 from the fact that main depth is 1 less due to redundancy of first level (depth in number of change of level) + if self.end & (1 << ix) != 0 { + // this is a skip + nb_skip += 1; + } else { + // continue only if right (like first if condition) + if index & (1 << ix) == 0 { + break; + } + } + } + self.depth - nb_skip + } else { + self.depth + } + } + + /// resolve the tree path for a given index. + pub fn path_node_key>(&self, index: usize) -> KN { + let tmp = (!0usize << (self.depth - self.end_depth)) | (index >> self.end_depth); + if !tmp == 0 { + let mut result: KN = (index, self.depth).into(); + for i in 0..self.end_depth { + let ix = self.end_depth - i - 1; // - 1 from the fact that main depth is 1 less due to redundancy of first level (depth in number of change of level) + if self.end & (1 << ix) != 0 { + // this is a skip + let ix = result.depth() - ix - 1; + result.remove_at(ix); + } else { + // continue only if right (like first if condition) + if index & (1 << ix) == 0 { + break; + } + } + } + result + } else { + (index, self.depth).into() + } + } + + pub fn iter_depth(&self, from: Option) -> impl Iterator { + if let Some(from) = from { + unimplemented!(); + } + let nb_elements = self.nb_elements(); + let mut index = 0; + let mut depth = self.depth; + let length = self.length; + let mut end = UsizeKeyNode::from((self.end, self.end_depth)); + let mut next_skip = length - if end.depth > 0 { + 1usize << end.depth // two time deletion range + } else { + 0 + }; + crate::rstd::iter::from_fn(move || { + if index < nb_elements { + if index == next_skip { + while end.pop_front() == Some(true) { + depth -= 1; + } + while end.nibble_at(0) == Some(false) { + end.pop_front(); + } + if end.depth > 0 { + next_skip += 1usize << end.depth + } + } + index += 1; + Some(depth) + } else { + None + } + }) + } + pub fn iter_path_node_key(&self, from: Option) -> impl Iterator + where + KN: KeyNode + From<(usize, usize)> + Clone, + { + if let Some(from) = from { + unimplemented!(); + } + let nb_elements = self.nb_elements(); + // TODO index should not be use but key.value, this is double counting things + let mut index = 0; + let length = self.length; + let mut end = KN::from((self.end, self.end_depth)); + let mut next_skip = length - if end.depth() > 0 { + 1usize << end.depth() // two time deletion range + } else { + 0 + }; + let mut key: KN = (0, self.depth).into(); + crate::rstd::iter::from_fn(move || { + if index < nb_elements { + if index == next_skip { + while end.pop_front() == Some(true) { + let ix = key.depth() - end.depth() - 1; + key.remove_at(ix); + } + while end.nibble_at(0) == Some(false) { + end.pop_front(); + } + if end.depth() > 0 { + next_skip += 1usize << end.depth() + } + } + let result = key.clone(); + key.increment_no_increase(); + index += 1; + Some(result) + } else { + None + } + }) + } + + fn pop(&mut self, nb: usize) { + unimplemented!("update max depth"); + } + + fn pop_front(&mut self, nb: usize) { + unimplemented!("update max depth"); + // TODO if start = max_depth_length / 2 -> max_depth - 1 + } + + fn max_depth_length(end: &usize) -> usize { + // 2^x = max_depth_length + unimplemented!() + } + + fn front_depth(index: usize) -> usize { + unimplemented!("for index between end and max len"); + } + + fn tail_depth(index: usize) -> usize { + unimplemented!("for index between end and max len"); + } +} + + + +// prefix scheme, the prefix use to avoid conflict of hash in a single trie is build +// upon indexed key of the leftmost child with the depth of the prefix and then the compact encoding. +// Therefore it can be use to iterate if there is only a single state for the trie. +// +// prefix scheme: not two node with same prefix ++ hash. +// meta & root cannot happen. +// level 1 can happen: just prefix 0 or 1 +// level 2 with level 1 can happen but only on different prefix +// level 3 with level 1 +// +// no offset and the depth of , therefore compact encoding is rather suitable for it. +// We use a compact +// +// NOTE that changing trie changes depth (existing k at depth 2 moving to depth 4), therefore the scheme is rather broken +// as we cannot address the nodes anymore. +// Therefore we should prefix over the index(key) as compact. For intermediattory key it will be +// the leftmost key index. TODO make test to check no collision and write asumption that to create +// collision we need inline values of length == to hash (to target previous 2 values hash eg for 3 +// nodes trie: hash(v1,v2) = h1, v3 = h1 but this implies h1 of length of hash and this means that +// we hash the value (with inline hash of length being strictly the hash length this can be use: +// CONCLUSION even if we cannot run inline values of length of the H::Out (more should be fine as +// it implies a second round of hashing) -> can be avoided with custom encoder. +// Inline value less than size hash are a problem on the other hand: when close to size hash we can +// find collision rather easilly, but that does not work because leftmost index is 3 for v3 and 1 +// for h1 so seems rather safe. If removed from start (offset), then it is not written so safe to +// except V1 del then V2 become h(v1,v2) and then v3 = v2 does break but prefix do not move : v2 is +// still 2 and v3 is still 3 so fine to. +// Could add a value bool to the prefix or the compact encoding scheme to indicate that it is a +// terminal value -> no value are stored outside? -> seems good to iterate (same for terminal node +// with a inline value -> 4 info here : intermediate, first value, second value, both value (the +// three lasts being the same (first in fact). This lead to possible iteration by. +// For partial storage we can use same approach for a few level of intermediate (this will bound +// key size for fix prefix, then last value is reserved for compact encoding of level next which +// should really never happen). +// +// NOTE inline value does not make sense, api should only use hash, additional api could store +// access values from terminal hash. +// Prefix wise, we could store in same db with key as prefix. Also if we want to inline value, +// then the value just need to be extract from terminal hash instead. (terminal hash marker +// and value describe above is still interesting). + + +/// key of node is a sequence of one bit nibbles. +pub trait KeyNode { + fn depth(&self) -> usize; + // return nibble at depth (bin tree so return bool) + fn nibble_at(&self, depth: usize) -> Option; + // last is leaf + fn pop_back(&mut self) -> Option; + fn push_back(&mut self, nibble: bool); + fn pop_front(&mut self) -> Option; + fn push_front(&mut self, nibble: bool); + fn remove_at(&mut self, depth: usize); + fn increment_no_increase(&mut self); + fn starts_with(&self, other: &Self) -> bool; + fn common_depth(&self, other: &Self) -> usize; +} + +#[cfg(test)] +#[derive(Clone, Debug)] +// please do not use, only for test of (usize, K) +struct VecKeyNode(std::collections::VecDeque); +#[cfg(test)] +impl KeyNode for VecKeyNode { + fn increment_no_increase(&mut self) { + for i in (0..self.0.len()).rev() { + match self.0.get_mut(i) { + Some(v) => { + if !*v { + *v = true; + break; + } + }, + None => { + unreachable!("should only be call when guaranties to not increase depth"); + }, + } + } + } + fn depth(&self) -> usize { + self.0.len() + } + fn nibble_at(&self, depth: usize) -> Option { + self.0.get(depth).cloned() + } + fn pop_back(&mut self) -> Option { + self.0.pop_back() + } + fn push_back(&mut self, nibble: bool) { + self.0.push_back(nibble) + } + fn pop_front(&mut self) -> Option { + self.0.pop_front() + } + fn push_front(&mut self, nibble: bool) { + self.0.push_front(nibble) + } + fn remove_at(&mut self, index: usize) { + self.0.remove(index); + } + fn starts_with(&self, other: &Self) -> bool { + // clone but it is test method only. + let mut tr = self.0.clone(); + tr.truncate(other.0.len()); + tr == other.0 + } + fn common_depth(&self, other: &Self) -> usize { + let bound = crate::rstd::cmp::min(self.0.len(), other.0.len()); + let mut depth = 0; + for i in 0..bound { + if self.0[i] == other.0[i] { + depth += 1; + } else { + break; + } + } + depth + } +} + +#[cfg(test)] +impl From<(usize, usize)> for VecKeyNode { + fn from((key, depth): (usize, usize)) -> Self { + if depth == 0 { + return VecKeyNode(std::collections::VecDeque::new()); + } +/* if depth == 0 { + return vec![]; + return vec![0]; + return vec![0, 0]; + ... + } + if depth == 1 { + return vec![1]; + return vec![0, 1]; + return vec![0, 0, 1]; + } + if depth == 2 { + return vec![1, 0]; + return vec![0, 1, 0]; + } + if depth == 3 { + return vec![1, 1]; + return vec![0, 1, 1]; + } + if depth == 4 { + return vec![1, 0, 0]; + } + if depth == 5 { + return vec![1, 0, 1]; + } +*/ + + + VecKeyNode( + (1..=depth).map(|i| right_at(key, depth - i)).collect() + ) + } +} + +#[cfg(test)] +impl Into for VecKeyNode { + fn into(self) -> usize { + let mut result = 0; + let depth = self.depth(); + self.0.into_iter().enumerate().for_each(|(i, b)| if b { + result = result | (1 << depth - (i + 1)); + }); + result + } +} + +#[derive(Clone, Copy, Debug)] +pub struct UsizeKeyNode { + value: usize, + depth: usize, +} + +// first is len, second is key +impl KeyNode for UsizeKeyNode { + fn depth(&self) -> usize { + self.depth + } + fn increment_no_increase(&mut self) { + self.value += 1; + } + fn nibble_at(&self, depth: usize) -> Option { + if depth < self.depth { + Some(right_at(self.value, self.depth - 1 - depth)) + } else { + None + } + } + fn pop_back(&mut self) -> Option { + if self.depth == 0 { + return None; + } + // TODO is pop returned value of any use: + // most likely not -> change trait and test + let result = self.value & 1; + self.depth -= 1; + self.value = self.value >> 1; + Some(result != 0) + } + fn push_back(&mut self, nibble: bool) { + self.value = self.value << 1; + self.value = self.value | 1; + self.depth +=1; + } + fn pop_front(&mut self) -> Option { + if self.depth == 0 { + return None; + } + // TODO is pop returned value of any use: + // most likely not -> change trait and test + let result = self.value & (1 << (self.depth - 1)); + self.value = self.value & !(1 << (self.depth - 1)); + self.depth -= 1; + Some(result != 0) + } + fn push_front(&mut self, nibble: bool) { + self.depth += 1; + self.value = self.value | (1 << (self.depth - 1)); + } + + fn remove_at(&mut self, index: usize) { + if index >= self.depth { + return; + } + if index == 0 { + self.pop_front(); + return; + } + if index == self.depth - 1 { + self.pop_back(); + return; + } + let right = self.value & !(!0usize << self.depth - index); + self.value = self.value & (!0usize << self.depth - index); + self.value = self.value >> 1; + self.depth -= 1; + self.value = self.value | right; + } + + fn starts_with(&self, other: &Self) -> bool { + if self.depth < other.depth { + false + } else { + self.value >> (self.depth - other.depth) == other.value + } + } + fn common_depth(&self, other: &Self) -> usize { + let (big, small) = if self.depth < other.depth { + (other, self) + } else { + (self, other) + }; + // end is not common + let big_v = big.value >> (big.depth - small.depth); + let diff = big_v ^ small.value; + + small.depth - (0usize.leading_zeros() - diff.leading_zeros()) as usize + } +} + +impl From<(usize, usize)> for UsizeKeyNode { + fn from((value, depth): (usize, usize)) -> Self { + // let value = value & !((!0) << depth); + UsizeKeyNode { value, depth } + } +} + +impl Into for UsizeKeyNode { + fn into(self) -> usize { + self.value + } +} + +#[test] +fn key_node_test() { + let test = |start: usize, end: bool| { + let depth = depth(start); + let mut v = VecKeyNode::from((start, depth)); + let mut u = UsizeKeyNode::from((start, depth)); + assert_eq!(v.nibble_at(start), u.nibble_at(start)); + if !end { + assert_eq!(u.push_back(true), v.push_back(true)); + } + assert_eq!(u.pop_back(), v.pop_back()); + assert_eq!(u.push_front(true), v.push_front(true)); + assert_eq!(u.pop_front(), v.pop_front()); + if !end { + assert_eq!(start, u.into()); + assert_eq!(start, v.clone().into()); + } + assert_eq!(u.pop_back(), v.pop_back()); + let u: usize = u.into(); + assert_eq!(u, v.into()); + }; + let t: VecKeyNode = (5, 4).into(); + let t: Vec = t.0.into_iter().collect(); + assert_eq!(t, vec![false, true, false, true]); + let t: std::collections::VecDeque = [false, true, false, true].iter().cloned().collect(); + assert_eq!(5usize, VecKeyNode(t).into()); + for i in 0..17 { + test(i, false); + } + test(usize::max_value() - 1, true); +// test(usize::max_value()); +} + +/// A buffer for binary hasher of size 64. +pub struct Buffer64([u8; 64]); +impl AsRef<[u8]> for Buffer64 { + fn as_ref(&self) -> &[u8] { + &self.0[..] + } +} +impl AsMut<[u8]> for Buffer64 { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0[..] + } +} +impl Default for Buffer64 { + fn default() -> Self { + Buffer64([0; 64]) + } +} +/// Test function to use on every binary buffer implementation. +pub fn test_binary_hasher() { + let size = ::LENGTH * 2; + let buf = ::Buffer::default(); + assert_eq!(buf.as_ref().len(), size); + let null_hash = H::hash(&[]); + assert_eq!(H::NULL_HASH, null_hash.as_ref()); + +} + +pub trait ProcessNode { + /// Callback for an empty trie, return byte representation + /// of the hash for the empty trie. + fn process_empty_trie(&mut self) -> &[u8]; + /// Process two child node to produce the parent one. + fn process(&mut self, key: &KN, child1: &[u8], child2: &[u8]) -> HO; + /// callback on the calculated root. + fn register_root(&mut self, root: &HO); +} + +pub trait ProcessNodeProof: ProcessNode { + fn register_proof_hash(&mut self, hash: &HO); +} + +/// Does only proccess hash on its buffer. +/// Buffer length need to be right and is unchecked. +pub struct HashOnly<'a, H>(&'a mut [u8], PhantomData); + +impl<'a, H: BinaryHasher> HashOnly<'a, H> { + pub fn new(buff: &'a mut H::Buffer) -> Self { + HashOnly(buff.as_mut(), PhantomData) + } + pub fn new_unchecked(buff: &'a mut [u8]) -> Self { + HashOnly(buff, PhantomData) + } +} + +impl<'a, H: BinaryHasher, KN> ProcessNode for HashOnly<'a, H> { + fn process_empty_trie(&mut self) -> &[u8] { + H::NULL_HASH + } + fn process(&mut self, _key: &KN, child1: &[u8], child2: &[u8]) -> H::Out { + // Should use lower level trait than Hasher to avoid copies. + self.0[..H::LENGTH].copy_from_slice(child1); + self.0[H::LENGTH..].copy_from_slice(child2); + H::hash(&self.0[..]) + } + fn register_root(&mut self, _root: &H::Out) { } +} + +/// Buffer length need to be right and is unchecked, proof elements are +/// stored in memory (no streaming). +pub struct HashProof<'a, H: Hasher, I, KN>{ + buffer: &'a mut [u8], + // I must guaranty right depth and index in range regarding + // to the tree struct!!! + to_prove: I, + state: MultiProofState, + additional_hash: Vec, +} + +// We need some read ahead to manage state. +struct MultiProofState { + current_key: Option, + next_key1: Option, + next_key2: Option, + // going up a join key means droping current in favor of stack, + // if stack empty move forward instead. + join1: Option, + join2: Option, + stack: smallvec::SmallVec<[(KN, usize);4]>, + is_empty: bool, +} + +enum MultiProofResult { + RegisterLeft, + RegisterRight, + DoNothing, +} + +impl MultiProofState { + fn new(next_keys: &mut impl Iterator) -> Self { + let mut result = MultiProofState { + current_key: None, + next_key1: None, + next_key2: None, + join1: None, + join2: None, + stack: Default::default(), + is_empty: false, + }; + result.current_key = next_keys.next(); + if result.current_key.is_none() { + result.is_empty = true; + } + result.next_key1 = next_keys.next(); + result.next_key2 = next_keys.next(); + result.refresh_join1(); + result.refresh_join2(); + result + } + fn refresh_join1(&mut self) { + self.join1 = self.current_key.as_ref() + .and_then(|c| self.next_key1.as_ref().map(|n| n.common_depth(c))); + } + fn refresh_join2(&mut self) { + self.join2 = self.next_key1.as_ref() + .and_then(|n1| self.next_key2.as_ref().map(|n2| n2.common_depth(n1))); + } + + fn new_key(&mut self, key: &KN, next_keys: &mut impl Iterator) -> MultiProofResult { + let depth = key.depth(); + let start_with_current = self.current_key.as_ref().map(|c| c.starts_with(key)).unwrap_or(false); + if start_with_current { + // join management + if Some(depth) == self.join1 { + if let Some(join2) = self.join2 { + let stack_join = self.stack.last().map(|s| s.1); + if stack_join.map(|sj| join2 > sj).unwrap_or(true) { + // move fw, keep current. + // next_1 is dropped. + self.next_key1 = self.next_key2.take(); + self.refresh_join1(); + self.next_key2 = next_keys.next(); + self.refresh_join2(); + return MultiProofResult::DoNothing; + } + } + // from stack + if let Some((stack_hash, _stack_join)) = self.stack.pop() { + // current is dropped. + self.current_key = Some(stack_hash); + // TODO check if stack depth == this new depth (should be?). + self.refresh_join1(); + } else { + // fuse last interval + self.join1 = None; + self.join2 = None; + self.next_key1 = None; + self.next_key2 = None; + } + return MultiProofResult::DoNothing; + } else { + // no matching join1 depth exclude sibling case. + if self.current_key.as_ref() + .expect("start_with_current").nibble_at(key.depth()).expect("starts with") { + return MultiProofResult::RegisterLeft; + } else { + return MultiProofResult::RegisterRight; + } + } + } + + let start_with_next = self.next_key1.as_ref().map(|n| n.starts_with(key)).unwrap_or(false); + // next interval management + if start_with_next { + let mut sibling = false; + if let Some(join2) = self.join2 { + if join2 == depth { + // next is sibling, skip it and do not register. + // next2 is dropped + self.next_key2 = next_keys.next(); + self.refresh_join2(); + sibling = true; + } + } + + let right = self.next_key1.as_ref() + .expect("start_with_current").nibble_at(key.depth()).expect("starts with"); + if let Some(join1) = self.join1 { + if let Some(join2) = self.join2 { + if join2 > join1 { + // shift and stack + self.stack.push((self.current_key.take().expect("no next without current"), join1)); + self.current_key = self.next_key1.take(); + self.next_key1 = self.next_key2.take(); + self.next_key2 = next_keys.next(); + self.refresh_join1(); // TODO could also use join2 for join1 would be fastest + self.refresh_join2(); + } else { + // keep interval + } + } else { + // no next_key2, keep interval + } + } else { + unreachable!("next is defined (start_with_next)"); + } + if !sibling { + // TODO could skip right resolution in sibling case + if right { + return MultiProofResult::RegisterLeft; + } else { + return MultiProofResult::RegisterRight; + } + } + } + MultiProofResult::DoNothing + } +} + +impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> HashProof<'a, H, I, KN> { + // TODO write function to build from iter of unchecked usize indexes: map iter + // with either depth_at or the depth_iterator skipping undesired elements (second + // seems better as it filters out of range. + pub fn new(buff: &'a mut H::Buffer, mut to_prove: I) -> Self { + let state = MultiProofState::new(&mut to_prove); + HashProof { + buffer: buff.as_mut(), + to_prove, + state, + additional_hash: Vec::new(), + } + } + pub fn take_additional_hash(&mut self) -> Vec { + crate::rstd::mem::replace(&mut self.additional_hash, Vec::new()) + } +} + +impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> ProcessNode for HashProof<'a, H, I, KN> { + fn process_empty_trie(&mut self) -> &[u8] { + H::NULL_HASH + } + fn process(&mut self, key: &KN, child1: &[u8], child2: &[u8]) -> H::Out { + match self.state.new_key(key, &mut self.to_prove) { + MultiProofResult::DoNothing => (), + MultiProofResult::RegisterLeft => { + let mut to_push = H::Out::default(); + to_push.as_mut().copy_from_slice(child1); + self.additional_hash.push(to_push); + }, + MultiProofResult::RegisterRight => { + let mut to_push = H::Out::default(); + to_push.as_mut().copy_from_slice(child2); + self.additional_hash.push(to_push); + }, + } + + self.buffer[..H::LENGTH].copy_from_slice(child1); + self.buffer[H::LENGTH..].copy_from_slice(child2); + H::hash(&self.buffer[..]) + } + fn register_root(&mut self, root: &H::Out) { + if self.state.is_empty { + self.additional_hash.push(root.clone()); + } + } +} + +// This only include hash, for including hashed value or inline node, just map the process over the +// input iterator (note that for inline node we need to attach this inline info to the tree so it +// only make sense for small trie or fix length trie). +/// Returns a calculated hash +pub fn trie_root(layout: &SequenceBinaryTree, input: I, callback: &mut F) -> HO + where + HO: Default + AsRef<[u8]> + AsMut<[u8]>, + KN: KeyNode + Into + From<(usize, usize)> + Clone, + I: Iterator, + F: ProcessNode, +{ + debug_assert!(layout.start == 0, "unimplemented start"); + let mut iter = input.into_iter().zip(layout.iter_path_node_key::(None)).enumerate(); + debug_assert!({ + let (r, s) = iter.size_hint(); + if s == Some(r) { + layout.nb_elements() == r + } else { + true + } + }); + let mut depth1 = layout.depth; + let mut child1 = if let Some((_, (child, key))) = iter.next() { + debug_assert!(key.depth() == depth1); + child + } else { + debug_assert!(layout.nb_elements() == 0); + let mut result = HO::default(); + result.as_mut().copy_from_slice(callback.process_empty_trie()); + return result; + }; + debug_assert!(layout.depth_index(0) == layout.depth); + // use a stack that match 16 element without allocation, that is 4 element depth + let mut stack = smallvec::SmallVec::<[(HO, usize);4]>::new(); + let mut key: KN = (0, depth1).into(); + loop { + let last_stack_depth = stack.last().map(|e|e.1); + if Some(depth1) == last_stack_depth { + // process over stack + let (child2, _depth2) = stack.pop().expect("checked above"); + key.pop_back(); + // stacked on at left + let parent = callback.process(&key, child2.as_ref(), child1.as_ref()); + depth1 = key.depth(); + child1 = parent; + } else { + if let Some((index, (child2, key2))) = iter.next() { + key = key2; + if key.depth() == depth1 { + key.pop_back(); + // iter one at right + let parent = callback.process(&key, child1.as_ref(), child2.as_ref()); + depth1 = key.depth(); + child1 = parent; + } else { + stack.push((child1, depth1)); + child1 = child2; + depth1 = key.depth(); + } + } else { + break; + } + } + } + debug_assert!(stack.is_empty()); + callback.register_root(&child1); + child1 +} + +/// Returns a calculated hash +pub fn trie_root_from_proof( + layout: &SequenceBinaryTree, + input: I, + additional_hash: I2, + callback: &mut F, + allow_additionals_hashes: bool, +) -> Option + where + HO: Default + AsRef<[u8]> + AsMut<[u8]>, + KN: KeyNode + Into + From<(usize, usize)> + Clone, + I: IntoIterator, + I2: IntoIterator, + F: ProcessNode, +{ + if layout.nb_elements() == 0 { + if !allow_additionals_hashes && additional_hash.into_iter().next().is_some() { + return None; + } else { + let mut result = HO::default(); + result.as_mut().copy_from_slice(callback.process_empty_trie()); + return Some(result); + } + } + + let mut items = input.into_iter(); + let mut additional_hash = additional_hash.into_iter(); + let mut current; + if let Some(c) = items.next() { + current = c; + } else { + // TODO check if we even can produce such proof. + // no item case root is directly in additional + if let Some(h) = additional_hash.next() { + if allow_additionals_hashes || additional_hash.next().is_none() { + callback.register_root(&h); + return Some(h); + } + } + return None; + } + let mut next = items.next(); + let calc_common_depth = |current: &(KN, HO), next: &Option<(KN, HO)>| { + next.as_ref().map(|n| current.0.common_depth(&n.0)) + }; + let mut common_depth = calc_common_depth(¤t, &next); + let mut stack = smallvec::SmallVec::<[((KN, HO), usize);4]>::new(); + + while let Some(right) = current.0.pop_back() { + let depth = current.0.depth(); + if Some(depth) == stack.last().as_ref().map(|s| s.1) { + let ((stack_key, stack_hash), _stack_depth) = stack.pop().expect("tested in condition"); + debug_assert!(right == true); + debug_assert!(stack_key.starts_with(¤t.0) && current.0.starts_with(&stack_key)); + current.1 = callback.process(¤t.0, stack_hash.as_ref(), current.1.as_ref()); + continue; + } + if Some(depth) == common_depth { + let (key_next, hash_next) = next.take().expect("common depth is some"); + stack.push((current, depth)); // TODO process sibling without stack? or all on stack + current = (key_next, hash_next); + next = items.next(); + common_depth = calc_common_depth(¤t, &next); + continue; + } + if let Some(other) = additional_hash.next() { + if right { + current.1 = callback.process(¤t.0, other.as_ref(), current.1.as_ref()); + } else { + current.1 = callback.process(¤t.0, current.1.as_ref(), other.as_ref()); + } + } else { + return None; + } + } + + debug_assert!(current.0.depth() == 0); + if !allow_additionals_hashes && additional_hash.next().is_some() { + None + } else { + callback.register_root(¤t.1); + Some(current.1) + } +} + +#[cfg(test)] +mod test { + use keccak_hasher::KeccakHasher; + use super::*; + //use keccak_hasher::FixKeccakHasher; + + //type Tree = super::SequenceBinaryTree; + type Tree = super::SequenceBinaryTree; + + #[test] + fn test_max_depth() { + let values = [ + (0, 0), + (1, 0), + (2, 1), + (3, 2), + (4, 2), + (5, 3), + (8, 3), + (9, 4), + (16, 4), + (17, 5), + (32, 5), + ]; + let mut tree = Tree::default(); + let mut prev = 0; + for (nb, depth) in values.iter().cloned() { + let inc = nb - prev; + prev = nb; + tree.push(inc); + assert_eq!(tree.depth, depth); + let tree2 = Tree::new(0, 0, nb); + assert_eq!(tree2.depth, depth); + assert_eq!(tree, tree2); + } + } + + #[test] + fn test_depth_index() { + // 8 trie + let tree = Tree::new(0, 0, 7); + assert_eq!(tree.depth_index(3), 3); + assert_eq!(tree.depth_index(4), 3); + assert_eq!(tree.depth_index(6), 2); + let tree = Tree::new(0, 0, 6); + assert_eq!(tree.depth_index(0), 3); + assert_eq!(tree.depth_index(3), 3); + assert_eq!(tree.depth_index(4), 2); + assert_eq!(tree.depth_index(5), 2); + let tree = Tree::new(0, 0, 5); + assert_eq!(tree.depth_index(3), 3); + assert_eq!(tree.depth_index(4), 1); + // 16 trie + let tree = Tree::new(0, 0, 12); + assert_eq!(tree.depth_index(7), 4); + assert_eq!(tree.depth_index(8), 3); + assert_eq!(tree.depth_index(11), 3); + let tree = Tree::new(0, 0, 11); + assert_eq!(tree.depth_index(7), 4); + assert_eq!(tree.depth_index(8), 3); + assert_eq!(tree.depth_index(9), 3); + assert_eq!(tree.depth_index(10), 2); + let tree = Tree::new(0, 0, 10); + assert_eq!(tree.depth_index(7), 4); + assert_eq!(tree.depth_index(8), 2); + assert_eq!(tree.depth_index(9), 2); + let tree = Tree::new(0, 0, 9); + assert_eq!(tree.depth_index(7), 4); + assert_eq!(tree.depth_index(8), 1); + // 32 trie TODO + } + + #[test] + fn test_depth_iter() { + // cases corresponding to test_depth, TODO add more +// let cases = [7, 6, 5, 12, 11, 10, 9]; + for nb in (0usize..16) { + let mut n = 0; + let tree = Tree::new(0, 0, nb); + for (i, (d, k)) in tree.iter_depth(None) + .zip(tree.iter_path_node_key::(None)) + .enumerate() { + n += 1; + assert_eq!(d, tree.depth_index(i)); + assert_eq!(d, k.depth); + let k2: UsizeKeyNode = tree.path_node_key(i); + assert_eq!(k.depth, k2.depth); + assert_eq!(k.value, k2.value); + } + assert_eq!(n, nb); + } + } + + impl BinaryHasher for KeccakHasher { + const NULL_HASH: &'static [u8] = &[197, 210, 70, 1, 134, 247, 35, 60, 146, + 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, + 250, 216, 4, 93, 133, 164, 112]; + type Buffer = Buffer64; + } + + #[test] + fn test_keccack_hasher() { + test_binary_hasher::() + } + + fn hashes(l: usize) -> Vec<[u8;32]> { + (0..l).map(|i| { + let mut hash = ::Out::default(); + let v = (i as u64).to_be_bytes(); + hash.as_mut()[..8].copy_from_slice(&v[..]); + hash + }).collect() + } + + fn base16_roots() -> Vec<[u8;32]> { + let hashes = hashes(16); + let mut result = Vec::<[u8;32]>::new(); + + let khash = |a: &[u8], b: &[u8]| { + let mut v = Vec::new(); + v.extend_from_slice(a); + v.extend_from_slice(b); + ::hash(v.as_ref()) + }; + let mut hash = ::Out::default(); + hash.as_mut()[..].copy_from_slice(KeccakHasher::NULL_HASH); + result.push(hash); + result.push(hashes[0].clone()); + let base2 = khash(hashes[0].as_ref(), hashes[1].as_ref()); + result.push(base2); + result.push(khash( + base2.as_ref(), + hashes[2].as_ref(), + )); + let base4 = khash( + base2.as_ref(), + khash(hashes[2].as_ref(), hashes[3].as_ref()).as_ref(), + ); + result.push(base4); + result.push(khash( + base4.as_ref(), + hashes[4].as_ref(), + )); + let base2 = khash(hashes[4].as_ref(), hashes[5].as_ref()); + result.push(khash( + base4.as_ref(), + base2.as_ref(), + )); + result.push(khash( + base4.as_ref(), + khash( + base2.as_ref(), + hashes[6].as_ref(), + ).as_ref(), + )); + let base8 = khash( + base4.as_ref(), + khash( + base2.as_ref(), + khash(hashes[6].as_ref(), hashes[7].as_ref()).as_ref(), + ).as_ref(), + ); + result.push(base8); + result.push(khash( + base8.as_ref(), + hashes[8].as_ref(), + )); + let base2 = khash(hashes[8].as_ref(), hashes[9].as_ref()); + result.push(khash( + base8.as_ref(), + base2.as_ref(), + )); + result.push(khash( + base8.as_ref(), + khash( + base2.as_ref(), + hashes[10].as_ref(), + ).as_ref(), + )); + let base4 = khash( + base2.as_ref(), + khash(hashes[10].as_ref(), hashes[11].as_ref()).as_ref(), + ); + result.push(khash( + base8.as_ref(), + base4.as_ref(), + )); + result.push(khash( + base8.as_ref(), + khash( + base4.as_ref(), + hashes[12].as_ref(), + ).as_ref(), + )); + let base2 = khash(hashes[12].as_ref(), hashes[13].as_ref()); + result.push(khash( + base8.as_ref(), + khash( + base4.as_ref(), + base2.as_ref(), + ).as_ref(), + )); + result.push(khash( + base8.as_ref(), + khash( + base4.as_ref(), + khash( + base2.as_ref(), + hashes[14].as_ref(), + ).as_ref(), + ).as_ref(), + )); + result.push(khash( + base8.as_ref(), + khash( + base4.as_ref(), + khash( + base2.as_ref(), + khash(hashes[14].as_ref(), hashes[15].as_ref()).as_ref(), + ).as_ref(), + ).as_ref(), + )); + result + } + + #[test] + fn test_hash_only() { + let result = base16_roots(); + for l in 0..17 { + let tree = Tree::new(0, 0, l); + let mut hash_buf = ::Buffer::default(); + let mut callback = HashOnly::::new(&mut hash_buf); + let hashes: Vec<_> = hashes(l); + let root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.into_iter(), &mut callback); + assert_eq!(root.as_ref(), &result[l][..]); + } + } + + #[test] + fn test_one_element_proof() { + let result = base16_roots(); + for l in 0..17 { + let tree = Tree::new(0, 0, l); + let mut hash_buf = ::Buffer::default(); + let mut hash_buf2 = ::Buffer::default(); + let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); + let hashes: Vec<_> = hashes(l); + for p in 0..l { + let to_prove = vec![tree.path_node_key::(p)]; + let mut callback = HashProof::::new(&mut hash_buf, to_prove.into_iter()); + let root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); + assert_eq!(root.as_ref(), &result[l][..]); + let additional_hash = callback.take_additional_hash(); + let proof_items = vec![(tree.path_node_key::(p), hashes[p].clone())]; + let root = trie_root_from_proof::<_, UsizeKeyNode, _, _, _>( + &tree, + proof_items, + additional_hash.into_iter(), + &mut callback_read_proof, + false, + ); + let additional_hash = callback.additional_hash; + println!("{}, {}", l, p); + assert!(root.is_some()); + assert_eq!(root.unwrap().as_ref(), &result[l][..]); + } + } + } + + #[test] + fn test_multiple_elements_proof() { + let result = base16_roots(); + let tests = [ + (1, &[][..]), + (1, &[0][..]), + (4, &[][..]), + (4, &[1][..]), + (4, &[1, 2][..]), + (4, &[1, 2, 3][..]), + (13, &[1, 2, 3][..]), + (13, &[2, 3, 4][..]), + (13, &[2, 5][..]), + (13, &[2, 5, 11][..]), + (13, &[2, 11][..]), + (13, &[11, 12][..]), + (13, &[10, 12][..]), + (13, &[2, 11, 12][..]), + ]; + for (l, ps) in tests.iter() { + let l = *l; + let ps = *ps; + let tree = Tree::new(0, 0, l); + let mut hash_buf = ::Buffer::default(); + let mut hash_buf2 = ::Buffer::default(); + let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); + let hashes: Vec<_> = hashes(l); + let mut to_prove = Vec::new(); + let mut proof_items = Vec::new(); + for p in ps { + let p = *p; + to_prove.push(tree.path_node_key::(p)); + proof_items.push((tree.path_node_key::(p), hashes[p].clone())); + } + let mut callback = HashProof::::new(&mut hash_buf, to_prove.into_iter()); + let root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); + assert_eq!(root.as_ref(), &result[l][..]); + let additional_hash = callback.take_additional_hash(); + let root = trie_root_from_proof::<_, UsizeKeyNode, _, _, _>( + &tree, + proof_items, + additional_hash.into_iter(), + &mut callback_read_proof, + false, + ); + let additional_hash = callback.additional_hash; + println!("{}, {:?}", l, ps); + assert!(root.is_some()); + assert_eq!(root.unwrap().as_ref(), &result[l][..]); + } + } +} + +/// Small trait for to allow using buffer of type [u8; H::LENGTH * 2]. +pub trait BinaryHasher: Hasher { + /// Hash for the empty content (is hash(&[])). + const NULL_HASH: &'static [u8]; + type Buffer: AsRef<[u8]> + AsMut<[u8]> + Default; +} + +pub trait HasherComplex: BinaryHasher { + + /// Alternate hash with complex proof allowed + /// TODO expose buffer !! (then memory db use a single buf) + fn hash_complex< + I: Iterator::Out>>, + I2: Iterator::Out>, + >( + x: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> Option; +} + +impl HasherComplex for H { + fn hash_complex< + I: Iterator::Out>>, + I2: Iterator::Out>, + >( + x: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> Option { + let seq_trie = SequenceBinaryTree::new(0, 0, nb_children); + + let mut hash_buf2 = ::Buffer::default(); + let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); + let hash = if !proof { + // full node + let iter = children.filter_map(|v| v); // TODO assert same number as count + crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof) + } else { + // proof node + let iter_key = seq_trie.iter_depth(None).enumerate().map(Into::::into); + let iter = children + .zip(iter_key) + .filter_map(|(value, key)| if let Some(value) = value { + Some((key, value)) + } else { + None + }); + if let Some(hash) = crate::trie_root_from_proof( + &seq_trie, + iter, + additional_hashes, + &mut callback_read_proof, + false, + ) { + hash + } else { + return None; + } + }; + // TODO really need a real hash trait + let mut buf = Vec::with_capacity(x.len() + hash.as_ref().len()); + buf.extend_from_slice(x); + buf.extend_from_slice(hash.as_ref()); + Some(H::hash(buf.as_slice())) + } +} + +/// Same as HashDB but can modify the value upon storage, and apply +/// `HasherComplex`. +pub trait HashDBComplex: Send + Sync + HashDB { + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions + /// are counted and the equivalent number of `remove()`s must be performed before the data + /// is considered dead. + fn insert_complex< + I: Iterator>, + I2: Iterator, + >( + &mut self, + prefix: Prefix, + value: &[u8], + no_child_value: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> H::Out; +} diff --git a/test-support/keccak-hasher/Cargo.toml b/test-support/keccak-hasher/Cargo.toml index 5bfcde0d..1877e524 100644 --- a/test-support/keccak-hasher/Cargo.toml +++ b/test-support/keccak-hasher/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" tiny-keccak = "1.4.2" hash-db = { path = "../../hash-db", default-features = false, version = "0.15.2" } hash256-std-hasher = { path = "../../hash256-std-hasher", version = "0.15.2" } +ordered-trie = { path = "../../ordered-trie", default-features = false, version = "0.19.2"} [features] default = ["std"] diff --git a/test-support/keccak-hasher/src/lib.rs b/test-support/keccak-hasher/src/lib.rs index ed64612a..79a1330c 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/test-support/keccak-hasher/src/lib.rs @@ -15,6 +15,7 @@ //! Hasher implementation for the Keccak-256 hash use hash_db::Hasher; +//use hash_db::FixHash; use tiny_keccak::Keccak; use hash256_std_hasher::Hash256StdHasher; @@ -35,6 +36,41 @@ impl Hasher for KeccakHasher { } } +impl ordered_trie::BinaryHasher for KeccakHasher { + const NULL_HASH: &'static [u8] = &[197, 210, 70, 1, 134, 247, 35, 60, 146, + 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, + 250, 216, 4, 93, 133, 164, 112]; + type Buffer = ordered_trie::Buffer64; +} + +#[test] +fn test_keccack_hasher() { + ordered_trie::test_binary_hasher::() +} + +/* TODO this is rather bad trait see if delete?? +#[derive(Default, Debug, Clone, PartialEq)] +pub struct FixKeccakHasher([u8;32]); +impl FixHash for FixKeccakHasher { + type Hasher = KeccakHasher; + const NEED_FIRST_HASHED: bool = true; + const EMPTY_HASHES: &'static [&'static [u8]] = &[]; + + fn new(first: ::Out) -> Self { + FixKeccakHasher(first) + } + fn hash(&mut self, second: &::Out) { + unimplemented!() + } + fn current_state(&self) -> &::Out { + &self.0 + } + fn finalize(self) -> ::Out { + unimplemented!() + } +} +*/ + #[cfg(test)] mod tests { use super::*; diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index abc8801a..d2cdc6b3 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -13,6 +13,7 @@ hash256-std-hasher = { path = "../../hash256-std-hasher", version = "0.15.2" } keccak-hasher = { path = "../keccak-hasher", version = "0.15.2" } trie-db = { path = "../../trie-db", default-features = false, version = "0.19.2" } trie-root = { path = "../../trie-root", default-features = false, version = "0.15.2" } +ordered-trie = { path = "../../ordered-trie", default-features = false, version = "0.19.2"} parity-scale-codec = { version = "1.0.3", features = ["derive"] } [dev-dependencies] diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 5b717c83..a570f682 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -26,18 +26,21 @@ use trie_db::{ triedbmut::ChildReference, DBValue, trie_visit, - TrieBuilder, - TrieRoot, + TrieBuilderComplex, + TrieRootComplex, + TrieRootUnhashedComplex, Partial, + BinaryHasher, + EncodedNoChild, }; use std::borrow::Borrow; use keccak_hasher::KeccakHasher; pub use trie_db::{ - decode_compact, encode_compact, + decode_compact, encode_compact, HashDBComplexDyn, nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, Trie, TrieConfiguration, TrieDB, TrieDBIterator, TrieDBMut, TrieDBNodeIterator, TrieError, - TrieIterator, TrieLayout, TrieMut, + TrieIterator, TrieLayout, TrieMut, Bitmap, BITMAP_LENGTH, }; pub use trie_root::TrieStream; pub mod node { @@ -49,6 +52,7 @@ pub struct ExtensionLayout; impl TrieLayout for ExtensionLayout { const USE_EXTENSION: bool = true; + const COMPLEX_HASH: bool = true; type Hash = KeccakHasher; type Codec = ReferenceNodeCodec; } @@ -59,45 +63,18 @@ impl TrieConfiguration for ExtensionLayout { } /// generic hasher. pub struct GenericNoExtensionLayout(PhantomData); -impl TrieLayout for GenericNoExtensionLayout { +impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; + const COMPLEX_HASH: bool = true; type Hash = H; type Codec = ReferenceNodeCodecNoExt; } -impl TrieConfiguration for GenericNoExtensionLayout { } +impl TrieConfiguration for GenericNoExtensionLayout { } /// Trie layout without extension nodes. pub type NoExtensionLayout = GenericNoExtensionLayout; -/// Children bitmap codec for radix 16 trie. -pub struct Bitmap(u16); - -const BITMAP_LENGTH: usize = 2; - -impl Bitmap { - - fn decode(data: &[u8]) -> Result { - Ok(u16::decode(&mut &data[..]) - .map(|v| Bitmap(v))?) - } - - fn value_at(&self, i: usize) -> bool { - self.0 & (1u16 << i) != 0 - } - - fn encode>(has_children: I , output: &mut [u8]) { - let mut bitmap: u16 = 0; - let mut cursor: u16 = 1; - for v in has_children { - if v { bitmap |= cursor } - cursor <<= 1; - } - output[0] = (bitmap % 256) as u8; - output[1] = (bitmap / 256) as u8; - } -} - pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; pub type RefTrieDBNoExt<'a> = trie_db::TrieDB<'a, NoExtensionLayout>; pub type RefTrieDBMut<'a> = trie_db::TrieDBMut<'a, ExtensionLayout>; @@ -117,13 +94,13 @@ pub fn reference_trie_root(input: I) -> ::Out w trie_root::trie_root::(input) } -fn reference_trie_root_unhashed(input: I) -> Vec where +/*fn reference_trie_root_unhashed(input: I) -> Vec where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { trie_root::unhashed_trie::(input) -} +}*/ pub fn reference_trie_root_no_extension(input: I) -> ::Out where I: IntoIterator, @@ -133,12 +110,63 @@ pub fn reference_trie_root_no_extension(input: I) -> (input) } -fn reference_trie_root_unhashed_no_extension(input: I) -> Vec where +/*fn reference_trie_root_unhashed_no_extension(input: I) -> Vec where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { trie_root::unhashed_trie_no_extension::(input) +}*/ + +fn data_sorted_unique(input: I) -> Vec<(A, B)> + where + I: IntoIterator, +{ + let mut m = std::collections::BTreeMap::new(); + for (k,v) in input { + let _ = m.insert(k,v); // latest value for uniqueness + } + m.into_iter().collect() +} + +pub fn reference_trie_root_iter_build(input: I) -> ::Out where + I: IntoIterator, + A: AsRef<[u8]> + Ord + fmt::Debug, + B: AsRef<[u8]> + fmt::Debug, +{ + let mut cb = trie_db::TrieRootComplex::::default(); + trie_visit::(data_sorted_unique(input), &mut cb); + cb.root.unwrap_or(Default::default()) +} + +pub fn reference_trie_root_no_extension_iter_build(input: I) -> ::Out where + I: IntoIterator, + A: AsRef<[u8]> + Ord + fmt::Debug, + B: AsRef<[u8]> + fmt::Debug, +{ + let mut cb = trie_db::TrieRootComplex::::default(); + trie_visit::(data_sorted_unique(input), &mut cb); + cb.root.unwrap_or(Default::default()) +} + +fn reference_trie_root_unhashed_iter_build(input: I) -> Vec where + I: IntoIterator, + A: AsRef<[u8]> + Ord + fmt::Debug, + B: AsRef<[u8]> + fmt::Debug, +{ + let mut cb = trie_db::TrieRootUnhashedComplex::::default(); + trie_visit::(data_sorted_unique(input), &mut cb); + cb.root.unwrap_or(Default::default()) +} + +fn reference_trie_root_unhashed_no_extension_iter_build(input: I) -> Vec where + I: IntoIterator, + A: AsRef<[u8]> + Ord + fmt::Debug, + B: AsRef<[u8]> + fmt::Debug, +{ + let mut cb = trie_db::TrieRootUnhashedComplex::::default(); + trie_visit::(data_sorted_unique(input), &mut cb); + cb.root.unwrap_or(Default::default()) } const EMPTY_TRIE: u8 = 0; @@ -610,26 +638,17 @@ impl<'a> Input for ByteSliceInput<'a> { } } -// NOTE: what we'd really like here is: -// `impl NodeCodec for RlpNodeCodec where ::Out: Decodable` -// but due to the current limitations of Rust const evaluation we can't do -// `const HASHED_NULL_NODE: ::Out = ::Out( … … )`. -// Perhaps one day soon? -impl NodeCodec for ReferenceNodeCodec { - type Error = CodecError; - type HashOut = H::Out; - - fn hashed_null_node() -> ::Out { - H::hash(::empty_node()) - } - - fn decode_plan(data: &[u8]) -> ::std::result::Result { +impl ReferenceNodeCodec { + fn decode_plan_internal( + data: &[u8], + is_complex: bool, + ) -> ::std::result::Result<(NodePlan, usize), ::Error> { let mut input = ByteSliceInput::new(data); - match NodeHeader::decode(&mut input)? { - NodeHeader::Null => Ok(NodePlan::Empty), + let node = match NodeHeader::decode(&mut input)? { + NodeHeader::Null => NodePlan::Empty, NodeHeader::Branch(has_value) => { let bitmap_range = input.take(BITMAP_LENGTH)?; - let bitmap = Bitmap::decode(&data[bitmap_range])?; + let bitmap = Bitmap::decode(&data[bitmap_range]); let value = if has_value { let count = >::decode(&mut input)?.0 as usize; @@ -643,16 +662,20 @@ impl NodeCodec for ReferenceNodeCodec { ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - let count = >::decode(&mut input)?.0 as usize; - let range = input.take(count)?; - children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + if is_complex { + children[i] = Some(NodeHandlePlan::Inline(Range { start: 0, end: 0 })); } else { - NodeHandlePlan::Inline(range) - }); + let count = >::decode(&mut input)?.0 as usize; + let range = input.take(count)?; + children[i] = Some(if count == H::LENGTH { + NodeHandlePlan::Hash(range) + } else { + NodeHandlePlan::Inline(range) + }); + } } } - Ok(NodePlan::Branch { value, children }) + NodePlan::Branch { value, children } } NodeHeader::Extension(nibble_count) => { let partial = input.take( @@ -666,10 +689,10 @@ impl NodeCodec for ReferenceNodeCodec { } else { NodeHandlePlan::Inline(range) }; - Ok(NodePlan::Extension { + NodePlan::Extension { partial: NibbleSlicePlan::new(partial, partial_padding), child - }) + } } NodeHeader::Leaf(nibble_count) => { let partial = input.take( @@ -678,12 +701,35 @@ impl NodeCodec for ReferenceNodeCodec { let partial_padding = nibble_ops::number_padding(nibble_count); let count = >::decode(&mut input)?.0 as usize; let value = input.take(count)?; - Ok(NodePlan::Leaf { + NodePlan::Leaf { partial: NibbleSlicePlan::new(partial, partial_padding), value, - }) + } } - } + }; + Ok((node, input.offset)) + } +} + +// NOTE: what we'd really like here is: +// `impl NodeCodec for RlpNodeCodec where ::Out: Decodable` +// but due to the current limitations of Rust const evaluation we can't do +// `const HASHED_NULL_NODE: ::Out = ::Out( … … )`. +// Perhaps one day soon? +impl NodeCodec for ReferenceNodeCodec { + type Error = CodecError; + type HashOut = H::Out; + + fn hashed_null_node() -> ::Out { + H::hash(::empty_node()) + } + + fn decode_plan(data: &[u8]) -> ::std::result::Result { + Ok(Self::decode_plan_internal(data, false)?.0) + } + + fn decode_plan_no_child(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { + Self::decode_plan_internal(data, true) } fn is_empty_node(data: &[u8]) -> bool { @@ -722,7 +768,8 @@ impl NodeCodec for ReferenceNodeCodec { fn branch_node( children: impl Iterator>>>, maybe_value: Option<&[u8]>, - ) -> Vec { + mut register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild) { let mut output = vec![0; BITMAP_LENGTH + 1]; let mut prefix: [u8; 3] = [0; 3]; let have_value = if let Some(value) = maybe_value { @@ -731,44 +778,77 @@ impl NodeCodec for ReferenceNodeCodec { } else { false }; + let mut ix = 0; + let ix = &mut ix; + let mut register_children = register_children.as_mut(); + let register_children = &mut register_children; + let no_child = if register_children.is_some() { + EncodedNoChild::Range(Range { + start: 0, + end: output.len(), + }) + } else { + EncodedNoChild::Unused + }; let has_children = children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { + if let Some(ranges) = register_children { + // this assume scale codec put len on one byte, which is the + // case for reasonable hash length. + let encode_size_offset = 1; + ranges[*ix] = Some(Range { + start: output.len() + encode_size_offset, + end: output.len() + encode_size_offset + h.as_ref().len(), + }); + *ix += 1; + } h.as_ref().encode_to(&mut output); true } &Some(ChildReference::Inline(inline_data, len)) => { + if let Some(ranges) = register_children { + let encode_size_offset = 1; + ranges[*ix] = Some(Range { + start: output.len() + encode_size_offset, + end: output.len() + encode_size_offset + len, + }); + *ix += 1; + } inline_data.as_ref()[..len].encode_to(&mut output); true } - None => false, + None => { + if register_children.is_some() { + *ix += 1; + } + false + }, }); branch_node_buffered(have_value, has_children, prefix.as_mut()); output[0..BITMAP_LENGTH + 1].copy_from_slice(prefix.as_ref()); - output + (output, no_child) } fn branch_node_nibbled( _partial: impl Iterator, _number_nibble: usize, _children: impl Iterator>>>, - _maybe_value: Option<&[u8]>) -> Vec { + _maybe_value: Option<&[u8]>, + _register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild) { unreachable!() } } -impl NodeCodec for ReferenceNodeCodecNoExt { - type Error = CodecError; - type HashOut = ::Out; - - fn hashed_null_node() -> ::Out { - H::hash(::empty_node()) - } - - fn decode_plan(data: &[u8]) -> ::std::result::Result { +impl ReferenceNodeCodecNoExt { + fn decode_plan_internal( + data: &[u8], + is_complex: bool, + ) -> ::std::result::Result<(NodePlan, usize), ::Error> { let mut input = ByteSliceInput::new(data); - match NodeHeaderNoExt::decode(&mut input)? { - NodeHeaderNoExt::Null => Ok(NodePlan::Empty), + let node = match NodeHeaderNoExt::decode(&mut input)? { + NodeHeaderNoExt::Null => NodePlan::Empty, NodeHeaderNoExt::Branch(has_value, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) @@ -780,7 +860,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { )?; let partial_padding = nibble_ops::number_padding(nibble_count); let bitmap_range = input.take(BITMAP_LENGTH)?; - let bitmap = Bitmap::decode(&data[bitmap_range])?; + let bitmap = Bitmap::decode(&data[bitmap_range]); let value = if has_value { let count = >::decode(&mut input)?.0 as usize; Some(input.take(count)?) @@ -793,20 +873,24 @@ impl NodeCodec for ReferenceNodeCodecNoExt { ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - let count = >::decode(&mut input)?.0 as usize; - let range = input.take(count)?; - children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + if is_complex { + children[i] = Some(NodeHandlePlan::Inline(Range { start: 0, end: 0 })); } else { - NodeHandlePlan::Inline(range) - }); + let count = >::decode(&mut input)?.0 as usize; + let range = input.take(count)?; + children[i] = Some(if count == H::LENGTH { + NodeHandlePlan::Hash(range) + } else { + NodeHandlePlan::Inline(range) + }); + } } } - Ok(NodePlan::NibbledBranch { + NodePlan::NibbledBranch { partial: NibbleSlicePlan::new(partial, partial_padding), value, children, - }) + } } NodeHeaderNoExt::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; @@ -820,12 +904,30 @@ impl NodeCodec for ReferenceNodeCodecNoExt { let partial_padding = nibble_ops::number_padding(nibble_count); let count = >::decode(&mut input)?.0 as usize; let value = input.take(count)?; - Ok(NodePlan::Leaf { + NodePlan::Leaf { partial: NibbleSlicePlan::new(partial, partial_padding), value, - }) + } } - } + }; + Ok((node, input.offset)) + } +} + +impl NodeCodec for ReferenceNodeCodecNoExt { + type Error = CodecError; + type HashOut = ::Out; + + fn decode_plan(data: &[u8]) -> ::std::result::Result { + Ok(Self::decode_plan_internal(data, false)?.0) + } + + fn decode_plan_no_child(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { + Self::decode_plan_internal(data, true) + } + + fn hashed_null_node() -> ::Out { + H::hash(::empty_node()) } fn is_empty_node(data: &[u8]) -> bool { @@ -853,7 +955,8 @@ impl NodeCodec for ReferenceNodeCodecNoExt { fn branch_node( _children: impl Iterator::Out>>>>, _maybe_value: Option<&[u8]>, - ) -> Vec { + _register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild) { unreachable!() } @@ -862,7 +965,8 @@ impl NodeCodec for ReferenceNodeCodecNoExt { number_nibble: usize, children: impl Iterator>>>, maybe_value: Option<&[u8]>, - ) -> Vec { + mut register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild) { let mut output = if maybe_value.is_some() { partial_from_iterator_encode( partial, @@ -882,32 +986,67 @@ impl NodeCodec for ReferenceNodeCodecNoExt { if let Some(value) = maybe_value { value.encode_to(&mut output); }; + let mut ix = 0; + let ix = &mut ix; + let mut register_children = register_children.as_mut(); + let register_children = &mut register_children; + let no_child = if register_children.is_some() { + EncodedNoChild::Range(Range { + start: 0, + end: output.len(), + }) + } else { + EncodedNoChild::Unused + }; Bitmap::encode(children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { + if let Some(ranges) = register_children { + // this assume scale codec put len on one byte, which is the + // case for reasonable hash length. + let encode_size_offset = 1; + ranges[*ix] = Some(Range { + start: output.len() + encode_size_offset, + end: output.len() + encode_size_offset + h.as_ref().len(), + }); + *ix += 1; + } h.as_ref().encode_to(&mut output); true } &Some(ChildReference::Inline(inline_data, len)) => { + if let Some(ranges) = register_children { + let encode_size_offset = 1; + ranges[*ix] = Some(Range { + start: output.len() + encode_size_offset, + end: output.len() + encode_size_offset + len, + }); + *ix += 1; + } inline_data.as_ref()[..len].encode_to(&mut output); true } - None => false, + None => { + if register_children.is_some() { + *ix += 1; + } + false + }, }), bitmap.as_mut()); output[bitmap_index..bitmap_index + BITMAP_LENGTH] .copy_from_slice(&bitmap.as_ref()[..BITMAP_LENGTH]); - output + (output, no_child) } } /// Compare trie builder and in memory trie. -pub fn compare_implementations + Eq> ( +pub fn compare_implementations + ordered_trie::HashDBComplex + Eq> ( data: Vec<(Vec, Vec)>, mut memdb: X, mut hashdb: X, ) { let root_new = { - let mut cb = TrieBuilder::new(&mut hashdb); + let mut cb = TrieBuilderComplex::new(&mut hashdb); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -947,10 +1086,10 @@ pub fn compare_implementations + Eq> /// Compare trie builder and trie root implementations. pub fn compare_root( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB, + mut memdb: impl ordered_trie::HashDBComplex, ) { let root_new = { - let mut cb = TrieRoot::::default(); + let mut cb = TrieRootComplex::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -971,11 +1110,11 @@ pub fn compare_unhashed( data: Vec<(Vec, Vec)>, ) { let root_new = { - let mut cb = trie_db::TrieRootUnhashed::::default(); + let mut cb = trie_db::TrieRootUnhashedComplex::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; - let root = reference_trie_root_unhashed(data); + let root = reference_trie_root_unhashed_iter_build(data); assert_eq!(root, root_new); } @@ -986,11 +1125,11 @@ pub fn compare_unhashed_no_extension( data: Vec<(Vec, Vec)>, ) { let root_new = { - let mut cb = trie_db::TrieRootUnhashed::::default(); + let mut cb = trie_db::TrieRootUnhashedComplex::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; - let root = reference_trie_root_unhashed_no_extension(data); + let root = reference_trie_root_unhashed_no_extension_iter_build(data); assert_eq!(root, root_new); } @@ -1004,7 +1143,7 @@ pub fn calc_root( A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRoot::::default(); + let mut cb = TrieRootComplex::::default(); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1019,7 +1158,7 @@ pub fn calc_root_no_extension( A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRoot::::default(); + let mut cb = TrieRootComplex::::default(); trie_db::trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1033,9 +1172,9 @@ pub fn calc_root_build( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + DB: hash_db::HashDB + ordered_trie::HashDBComplex, { - let mut cb = TrieBuilder::new(hashdb); + let mut cb = TrieBuilderComplex::new(hashdb); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1050,9 +1189,9 @@ pub fn calc_root_build_no_extension( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + DB: hash_db::HashDB + ordered_trie::HashDBComplex, { - let mut cb = TrieBuilder::new(hashdb); + let mut cb = TrieBuilderComplex::new(hashdb); trie_db::trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1061,11 +1200,11 @@ pub fn calc_root_build_no_extension( /// This uses the variant without extension nodes. pub fn compare_implementations_no_extension( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB, - mut hashdb: impl hash_db::HashDB, + mut memdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, + mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, ) { let root_new = { - let mut cb = TrieBuilder::new(&mut hashdb); + let mut cb = TrieBuilderComplex::new(&mut hashdb); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1104,8 +1243,8 @@ pub fn compare_implementations_no_extension( /// ordering before running when trie_build expect correct ordering). pub fn compare_implementations_no_extension_unordered( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB, - mut hashdb: impl hash_db::HashDB, + mut memdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, + mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, ) { let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { @@ -1118,7 +1257,7 @@ pub fn compare_implementations_no_extension_unordered( t.root().clone() }; let root_new = { - let mut cb = TrieBuilder::new(&mut hashdb); + let mut cb = TrieBuilderComplex::new(&mut hashdb); trie_visit::(b_map.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1149,7 +1288,7 @@ pub fn compare_implementations_no_extension_unordered( /// its input test data. pub fn compare_no_extension_insert_remove( data: Vec<(bool, Vec, Vec)>, - mut memdb: impl hash_db::HashDB, + mut memdb: impl ordered_trie::HashDBComplex, ) { let mut data2 = std::collections::BTreeMap::new(); let mut root = Default::default(); diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index a1a3be83..ad81a1bc 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -12,6 +12,7 @@ log = "0.4" smallvec = "1.0.0" hash-db = { path = "../hash-db", default-features = false, version = "0.15.2"} hashbrown = { version = "0.6.3", default-features = false } +ordered-trie = { path = "../ordered-trie", default-features = false, version = "0.19.2"} rustc-hex = { version = "2.1.0", default-features = false, optional = true } [dev-dependencies] @@ -38,3 +39,4 @@ std = [ [[bench]] name = "bench" harness = false + diff --git a/trie-db/benches/bench.rs b/trie-db/benches/bench.rs index e2a63523..331b94e5 100644 --- a/trie-db/benches/bench.rs +++ b/trie-db/benches/bench.rs @@ -326,7 +326,7 @@ fn trie_mut_ref_root_a(c: &mut Criterion) { .map(|v| (&v.0, &v.1)) .collect::>(); - reference_trie::reference_trie_root(inputc); + reference_trie::reference_trie_root_iter_build(inputc); }), data); } @@ -346,7 +346,7 @@ fn trie_mut_ref_root_b(c: &mut Criterion) { .map(|v| (&v.0, &v.1)) .collect::>(); - reference_trie::reference_trie_root(inputc); + reference_trie::reference_trie_root_iter_build(inputc); }), data); } diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 4b9d0efc..c329ccfd 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -22,7 +22,7 @@ use reference_trie::{ ExtensionLayout, NoExtensionLayout, proof::{generate_proof, verify_proof}, - reference_trie_root, + reference_trie_root_iter_build as reference_trie_root, RefTrieDBMut, RefTrieDBMutNoExt, RefTrieDBNoExt, diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs index cf45448d..714e1b20 100644 --- a/trie-db/src/fatdbmut.rs +++ b/trie-db/src/fatdbmut.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use hash_db::{Hasher, EMPTY_PREFIX}; +use crate::node_codec::HashDBComplexDyn as HashDB; use super::{Result, DBValue, TrieDBMut, TrieMut, TrieLayout, TrieHash, CError}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index fa0cabeb..af72aaf6 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -17,13 +17,15 @@ //! implementation. //! See `trie_visit` function. -use hash_db::{Hasher, HashDB, Prefix}; -use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec}; +use hash_db::{Hasher, HashDB, Prefix, ComplexLayout}; +use ordered_trie::{HashDBComplex, HasherComplex}; +use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range}; use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; use crate::nibble::nibble_ops; -use crate::node_codec::NodeCodec; +use crate::node_codec::{NodeCodec, EncodedNoChild}; use crate::{TrieLayout, TrieHash}; +use crate::rstd::borrow::Borrow; macro_rules! exponential_out { (@3, [$($inpp:expr),*]) => { exponential_out!(@2, [$($inpp,)* $($inpp),*]) }; @@ -50,6 +52,15 @@ struct CacheAccum (Vec<(ArrayNode, Option, usize)>, Phan /// Initially allocated cache depth. const INITIAL_DEPTH: usize = 10; +#[inline] +fn register_children_buf() -> Option<[Option>; 16]> { + if T::COMPLEX_HASH { + Some(Default::default()) + } else { + None + } +} + impl CacheAccum where T: TrieLayout, @@ -134,7 +145,8 @@ impl CacheAccum &k2.as_ref()[..], k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); - let hash = callback.process(pr.left(), encoded, false); + let iter: Option<(EmptyIter>, _)> = None; + let hash = callback.process(pr.left(), (encoded, EncodedNoChild::Unused), false, iter); // insert hash in branch (first level branch only at this point) self.set_node(target_depth, nibble_value as usize, Some(hash)); @@ -195,19 +207,30 @@ impl CacheAccum // encode branch let v = self.0[last].1.take(); + + let mut register_children = register_children_buf::(); let encoded = T::Codec::branch_node( self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), + register_children.as_mut().map(|t| t.as_mut()), // TODO switch fully to codec approach or just return trim info ); - self.reset_depth(branch_d); let pr = NibbleSlice::new_offset(&key_branch, branch_d); - let branch_hash = callback.process(pr.left(), encoded, is_root && nkey.is_none()); + let branch_hash = if T::COMPLEX_HASH { + let len = self.0[last].0.as_ref().iter().filter(|v| v.is_some()).count(); + let children = self.0[last].0.as_ref().iter(); + callback.process(pr.left(), encoded, is_root && nkey.is_none(), Some((children, len))) + } else { + let iter: Option<(EmptyIter>, _)> = None; + callback.process(pr.left(), encoded, is_root && nkey.is_none(), iter) + }; + self.reset_depth(branch_d); if let Some(nkeyix) = nkey { let pr = NibbleSlice::new_offset(&key_branch, nkeyix.0); let nib = pr.right_range_iter(nkeyix.1); let encoded = T::Codec::extension_node(nib, nkeyix.1, branch_hash); - let h = callback.process(pr.left(), encoded, is_root); + let iter: Option<(EmptyIter>, _)> = None; + let h = callback.process(pr.left(), (encoded, EncodedNoChild::Unused), is_root, iter); h } else { branch_hash @@ -228,18 +251,29 @@ impl CacheAccum // encode branch let v = self.0[last].1.take(); let nkeyix = nkey.unwrap_or((0, 0)); + let mut register_children = register_children_buf::(); let pr = NibbleSlice::new_offset(&key_branch, nkeyix.0); let encoded = T::Codec::branch_node_nibbled( pr.right_range_iter(nkeyix.1), nkeyix.1, - self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref())); - self.reset_depth(branch_d); + self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), + register_children.as_mut().map(|t| t.as_mut()), // TODO switch fully to codec approach + ); let ext_length = nkey.as_ref().map(|nkeyix| nkeyix.0).unwrap_or(0); let pr = NibbleSlice::new_offset( &key_branch, branch_d - ext_length, ); - callback.process(pr.left(), encoded, is_root) + let result = if T::COMPLEX_HASH { + let len = self.0[last].0.as_ref().iter().filter(|v| v.is_some()).count(); + let children = self.0[last].0.as_ref().iter(); + callback.process(pr.left(), encoded, is_root, Some((children, len))) + } else { + let iter: Option<(EmptyIter>, _)> = None; + callback.process(pr.left(), encoded, is_root, iter) + }; + self.reset_depth(branch_d); + result } } @@ -297,15 +331,17 @@ pub fn trie_visit(input: I, callback: &mut F) &k2.as_ref()[..], k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); - callback.process(pr.left(), encoded, true); + let iter: Option<(EmptyIter>, _)> = None; + callback.process(pr.left(), (encoded, EncodedNoChild::Unused), true, iter); } else { depth_queue.flush_value(callback, last_depth, &previous_value); let ref_branches = previous_value.0; depth_queue.flush_branch(no_extension, callback, ref_branches, 0, true); } } else { + let iter: Option<(EmptyIter>, _)> = None; // nothing null root corner case - callback.process(hash_db::EMPTY_PREFIX, T::Codec::empty_node().to_vec(), true); + callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), EncodedNoChild::Unused), true, iter); } } @@ -318,7 +354,13 @@ pub trait ProcessEncodedNode { /// but usually it should be the Hash of encoded node. /// This is not something direcly related to encoding but is here for /// optimisation purpose (builder hash_db does return this value). - fn process(&mut self, prefix: Prefix, encoded_node: Vec, is_root: bool) -> ChildReference; + fn process( + &mut self, + prefix: Prefix, + encoded_node: (Vec, EncodedNoChild), + is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, + ) -> ChildReference; } /// Get trie root and insert visited node in a hash_db. @@ -336,13 +378,30 @@ impl<'a, H, HO, V, DB> TrieBuilder<'a, H, HO, V, DB> { } } +/// Get trie root and insert visited node in a hash_db. +/// As for all `ProcessEncodedNode` implementation, it +/// is only for full trie parsing (not existing trie). +pub struct TrieBuilderComplex<'a, H, HO, V, DB> { + db: &'a mut DB, + pub root: Option, + _ph: PhantomData<(H, V)>, +} + +impl<'a, H, HO, V, DB> TrieBuilderComplex<'a, H, HO, V, DB> { + pub fn new(db: &'a mut DB) -> Self { + TrieBuilderComplex { db, root: None, _ph: PhantomData } + } +} + + impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> for TrieBuilder<'a, H, ::Out, V, DB> { fn process( &mut self, prefix: Prefix, - encoded_node: Vec, + (encoded_node, _no_child): (Vec, EncodedNoChild), is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -359,6 +418,49 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> } } +impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode<::Out> + for TrieBuilderComplex<'a, H, ::Out, V, DB> { + fn process( + &mut self, + prefix: Prefix, + (encoded_node, no_child): (Vec, EncodedNoChild), + is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, + ) -> ChildReference<::Out> { + let len = encoded_node.len(); + if !is_root && len < ::LENGTH { + let mut h = <::Out as Default>::default(); + h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); + + return ChildReference::Inline(h, len); + } + + let hash = if let Some((children, nb_children)) = complex_hash { + let iter = children + .filter_map(|v| match v.borrow().as_ref() { + Some(ChildReference::Hash(v)) => Some(Some(v.clone())), + Some(ChildReference::Inline(v, _l)) => Some(Some(v.clone())), + None => None, + }); + self.db.insert_complex( + prefix, + &encoded_node[..], + no_child.encoded_no_child(&encoded_node[..]), + nb_children, + iter, + EmptyIter::default(), + false, + ) + } else { + self.db.insert(prefix, &encoded_node[..]) + }; + if is_root { + self.root = Some(hash.clone()); + }; + ChildReference::Hash(hash) + } +} + /// Calculate the trie root of the trie. pub struct TrieRoot { /// The resulting root. @@ -376,8 +478,9 @@ impl ProcessEncodedNode<::Out> for TrieRoot, + (encoded_node, _no_child): (Vec, EncodedNoChild), is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -394,6 +497,60 @@ impl ProcessEncodedNode<::Out> for TrieRoot { + /// The resulting root. + pub root: Option, + _ph: PhantomData, +} + +impl Default for TrieRootComplex { + fn default() -> Self { + TrieRootComplex { root: None, _ph: PhantomData } + } +} + +impl ProcessEncodedNode<::Out> for TrieRootComplex::Out> { + fn process( + &mut self, + _: Prefix, + (encoded_node, no_child): (Vec, EncodedNoChild), + is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, + ) -> ChildReference<::Out> { + let len = encoded_node.len(); + if !is_root && len < ::LENGTH { + let mut h = <::Out as Default>::default(); + h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); + + return ChildReference::Inline(h, len); + } + let hash = if let Some((children, nb_children)) = complex_hash { + let iter = children + .filter_map(|v| match v.borrow().as_ref() { + Some(ChildReference::Hash(v)) => Some(Some(v.clone())), + Some(ChildReference::Inline(v, _l)) => Some(Some(v.clone())), + None => None, + }); + ::hash_complex( + no_child.encoded_no_child(&encoded_node[..]), + nb_children, + iter, + EmptyIter::default(), + false, + ).expect("only proof fails: TODO two primitives") + } else { + ::hash(&encoded_node[..]) + }; + + if is_root { + self.root = Some(hash.clone()); + }; + ChildReference::Hash(hash) + } +} + + /// Get the trie root node encoding. pub struct TrieRootUnhashed { /// The resulting encoded root. @@ -407,6 +564,19 @@ impl Default for TrieRootUnhashed { } } +/// Get the trie root node encoding. +pub struct TrieRootUnhashedComplex { + /// The resulting encoded root. + pub root: Option>, + _ph: PhantomData, +} + +impl Default for TrieRootUnhashedComplex { + fn default() -> Self { + TrieRootUnhashedComplex { root: None, _ph: PhantomData } + } +} + #[cfg(feature = "std")] /// Calculate the trie root of the trie. /// Print a debug trace. @@ -428,8 +598,9 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, + (encoded_node, _no_child): (Vec, EncodedNoChild), is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { println!("Encoded node: {:x?}", &encoded_node); println!(" with prefix: {:x?}", &p); @@ -454,8 +625,9 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { fn process( &mut self, _: Prefix, - encoded_node: Vec, + (encoded_node, _no_child): (Vec, EncodedNoChild), is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -472,6 +644,47 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { } } +impl ProcessEncodedNode<::Out> for TrieRootUnhashedComplex { + fn process( + &mut self, + _: Prefix, + (encoded_node, no_child): (Vec, EncodedNoChild), + is_root: bool, + complex_hash: Option<(impl Iterator>>>, usize)>, + ) -> ChildReference<::Out> { + let len = encoded_node.len(); + if !is_root && len < ::LENGTH { + let mut h = <::Out as Default>::default(); + h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); + + return ChildReference::Inline(h, len); + } + let hash = if let Some((children, nb_children)) = complex_hash { + let iter = children + .filter_map(|v| match v.borrow().as_ref() { + Some(ChildReference::Hash(v)) => Some(Some(v.clone())), + Some(ChildReference::Inline(v, _l)) => Some(Some(v.clone())), + None => None, + }); + ::hash_complex( + no_child.encoded_no_child(&encoded_node[..]), + nb_children, + iter, + EmptyIter::default(), + false, + ).expect("only proof fails: TODO two primitives") + } else { + ::hash(&encoded_node[..]) + }; + + if is_root { + self.root = Some(encoded_node); + }; + ChildReference::Hash(hash) + } +} + + #[cfg(test)] mod test { use crate::DBValue; diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 05b52686..1d8f5d34 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -22,12 +22,15 @@ extern crate alloc; mod rstd { pub use std::{borrow, boxed, cmp, convert, fmt, hash, iter, marker, mem, ops, rc, result, vec}; pub use std::collections::VecDeque; + pub use std::collections::BTreeMap; pub use std::error::Error; + pub use std::iter::Empty as EmptyIter; } #[cfg(not(feature = "std"))] mod rstd { pub use core::{borrow, convert, cmp, iter, fmt, hash, marker, mem, ops, result}; + pub use core::iter::Empty as EmptyIter; pub use alloc::{boxed, rc, vec}; pub use alloc::collections::VecDeque; pub trait Error {} @@ -67,11 +70,12 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, Partial}; -pub use crate::iter_build::{trie_visit, ProcessEncodedNode, - TrieBuilder, TrieRoot, TrieRootUnhashed}; +pub use crate::node_codec::{NodeCodec, Partial, HashDBComplexDyn, EncodedNoChild, Bitmap, BITMAP_LENGTH}; +pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, + TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; pub use crate::iterator::TrieDBNodeIterator; pub use crate::trie_codec::{decode_compact, encode_compact}; +pub use ordered_trie::BinaryHasher; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; @@ -357,7 +361,7 @@ where /// Create new mutable instance of Trie. pub fn create( &self, - db: &'db mut dyn HashDB, + db: &'db mut dyn HashDBComplexDyn, root: &'db mut TrieHash, ) -> Box + 'db> { match self.spec { @@ -370,7 +374,7 @@ where /// Create new mutable instance of trie and check for errors. pub fn from_existing( &self, - db: &'db mut dyn HashDB, + db: &'db mut dyn HashDBComplexDyn, root: &'db mut TrieHash, ) -> Result + 'db>, TrieHash, CError> { match self.spec { @@ -392,8 +396,9 @@ pub trait TrieLayout { /// no partial in branch, if false the trie will only /// use branch and node with partials in both. const USE_EXTENSION: bool; + const COMPLEX_HASH: bool; /// Hasher to use for this trie. - type Hash: Hasher; + type Hash: BinaryHasher; /// Codec to use (needs to match hasher and nibble ops). type Codec: NodeCodec::Out>; } diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index ef50e8d6..d75af745 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -57,6 +57,16 @@ pub enum Node<'a> { NibbledBranch(NibbleSlice<'a>, [Option>; nibble_ops::NIBBLE_LENGTH], Option<&'a [u8]>), } +impl<'a> Node<'a> { + /// Check if this is a branch node plan. + pub fn is_branch(&self) -> bool { + match self { + Node::Branch(..) | Node::NibbledBranch(..) => true, + _ => false, + } + } +} + /// A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie /// node. This is used as a substructure of `NodePlan`. See `NodePlan` for details. #[derive(Debug, Clone, PartialEq, Eq)] @@ -172,6 +182,14 @@ impl NodePlan { }, } } + + /// Check if this is a branch node plan. + pub fn is_branch(&self) -> bool { + match self { + NodePlan::Branch{..} | NodePlan::NibbledBranch{..} => true, + _ => false, + } + } } /// An `OwnedNode` is an owned type from which a `Node` can be constructed which borrows data from diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index ad33ebf5..e6a514d2 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -19,8 +19,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec}; - +use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData, mem::replace}; /// Representation of a nible slice (right aligned). /// It contains a right aligned padded first byte (first pair element is the number of nibbles @@ -43,11 +42,23 @@ pub trait NodeCodec: Sized { /// Decode bytes to a `NodePlan`. Returns `Self::E` on failure. fn decode_plan(data: &[u8]) -> Result; + /// See `Decode_no_child`. + fn decode_plan_no_child(data: &[u8]) -> Result<(NodePlan, usize), Self::Error>; + /// Decode bytes to a `Node`. Returns `Self::E` on failure. fn decode(data: &[u8]) -> Result { + // TODO ensure real use codec have their own implementation + // as this can be slower Ok(Self::decode_plan(data)?.build(data)) } + /// Decode but child are not include (instead we put empty inline + /// nodes). + fn decode_no_child(data: &[u8]) -> Result<(Node, usize), Self::Error> { + let (plan, offset) = Self::decode_plan_no_child(data)?; + Ok((plan.build(data), offset)) + } + /// Check if the provided bytes correspond to the codecs "empty" node. fn is_empty_node(data: &[u8]) -> bool; @@ -72,7 +83,8 @@ pub trait NodeCodec: Sized { fn branch_node( children: impl Iterator>>>, value: Option<&[u8]>, - ) -> Vec; + register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild); /// Returns an encoded branch node with a possible partial path. /// `number_nibble` is the partial path length as in `extension_node`. @@ -80,6 +92,205 @@ pub trait NodeCodec: Sized { partial: impl Iterator, number_nibble: usize, children: impl Iterator>>>, - value: Option<&[u8]> - ) -> Vec; + value: Option<&[u8]>, + register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild); +} + +#[derive(Clone)] +pub enum EncodedNoChild { + // not runing complex + Unused, + // range over the full encoded + Range(Range), + // allocated in case we cannot use a range + Allocated(Vec), +} + +impl EncodedNoChild { + pub fn encoded_no_child<'a>(&'a self, encoded: &'a [u8]) -> &'a [u8] { + match self { + EncodedNoChild::Unused => encoded, + EncodedNoChild::Range(range) => &encoded[range.clone()], + EncodedNoChild::Allocated(data) => &data[..], + } + } + // TODO this is bad we should produce a branch that does + // not include it in the first place (new encode fn with + // default impl using trim no child). + pub fn trim_no_child(self, encoded: &mut Vec) { + match self { + EncodedNoChild::Unused => (), + EncodedNoChild::Range(range) => { + encoded.truncate(range.end); + if range.start != 0 { + *encoded = encoded.split_off(range.start); + } + }, + EncodedNoChild::Allocated(data) => { + replace(encoded, data); + }, + } + } + +} + +use ordered_trie::{HashDBComplex, HasherComplex}; +use hash_db::{HashDB, Prefix, HashDBRef, Hasher}; + +pub trait HashDBComplexDyn: Send + Sync + HashDB { + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions + /// are counted and the equivalent number of `remove()`s must be performed before the data + /// is considered dead. + /// + /// TODO warn semantic of children differs from HashDBComplex (in HashDBComplex it is the + /// children of the binary hasher, here it is the children of the patricia merkle trie). + fn insert_complex( + &mut self, + prefix: Prefix, + value: &[u8], + children: &[Option>], + no_child: EncodedNoChild, + ) -> H::Out; +} + +impl> HashDBComplexDyn for C { + fn insert_complex( + &mut self, + prefix: Prefix, + value: &[u8], + children: &[Option>], + no_child: EncodedNoChild, + ) -> H::Out { + + // TODO factor this with iter_build (just use the trait) + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let children = ComplexLayoutIterValues::new( + nb_children, + children.iter().filter_map(|v| v.as_ref()), + value, + ); + + >::insert_complex( + self, + prefix, + value, + no_child.encoded_no_child(value), + nb_children, + children, + EmptyIter::default(), + false, + ) + } +} + +impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDBComplexDyn { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + self.as_hash_db().get(key, prefix) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + self.as_hash_db().contains(key, prefix) + } +} + +impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDBComplexDyn { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + self.as_hash_db().get(key, prefix) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + self.as_hash_db().contains(key, prefix) + } +} + +// TODO this using a buffer is bad (we should switch +// binary hasher to use slice as input (or be able to)) +pub struct ComplexLayoutIterValues<'a, HO, I> { + nb_children: usize, + children: I, + node: &'a [u8], + _ph: PhantomData, +} +/* +code snippet for children iter: +ComplexLayoutIterValues::new(nb_children, children, value) + .map(|(is_defined, v)| { + debug_assert!(is_defined); + v + }); +code snippet for proof + let iter = ComplexLayoutIterValues::new(nb_children, children, value) + .zip(iter_key) + .filter_map(|((is_defined, hash), key)| if is_defined { + Some((key, hash)) + } else { + None + }); +*/ + +impl<'a, HO: Default, I> ComplexLayoutIterValues<'a, HO, I> { + pub fn new(nb_children: usize, children: I, node: &'a[u8]) -> Self { + ComplexLayoutIterValues { + nb_children, + children, + node, + _ph: PhantomData, + } + } +} + +impl<'a, HO: AsMut<[u8]> + Default, I: Iterator>> Iterator for ComplexLayoutIterValues<'a, HO, I> { + type Item = Option; + + fn next(&mut self) -> Option { + if let Some(range) = self.children.next() { + let range_len = range.len(); + if range_len == 0 { + // this is for undefined proof hash + return None; + } + let mut dest = HO::default(); + dest.as_mut()[..range_len].copy_from_slice(&self.node[range.clone()]); + /* inherent to default?? TODO consider doing it, but if refacto + * this will be part of hasher (when run with slice as input) + * for i in range_len..dest.len() { + dest[i] = 0; + }*/ + // TODO the input iterator is HO but could really be &HO, would need some + // change on trie_root to. + Some(Some(dest)) + } else { + None + } + } +} + +/// Children bitmap codec for radix 16 trie. +pub struct Bitmap(u16); + +/// Length of a 16 element bitmap. +pub const BITMAP_LENGTH: usize = 2; + +impl Bitmap { + + pub fn decode(data: &[u8]) -> Self { + let map = data[0] as u16 + data[1] as u16 * 256; + Bitmap(map) + } + + pub fn value_at(&self, i: usize) -> bool { + self.0 & (1u16 << i) != 0 + } + + pub fn encode>(has_children: I , output: &mut [u8]) { + let mut bitmap: u16 = 0; + let mut cursor: u16 = 1; + for v in has_children { + if v { bitmap |= cursor } + cursor <<= 1; + } + output[0] = (bitmap % 256) as u8; + output[1] = (bitmap / 256) as u8; + } } diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 817e9333..d5b16fbf 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -20,13 +20,15 @@ use crate::rstd::{ use hash_db::Hasher; +use crate::node_codec::Bitmap; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, Result as TrieResult, Trie, TrieError, TrieHash, TrieLayout, }; +use ordered_trie::BinaryHasher; -struct StackEntry<'a, C: NodeCodec> { +struct StackEntry<'a, C: NodeCodec, H> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, node: OwnedNode>, @@ -41,15 +43,20 @@ struct StackEntry<'a, C: NodeCodec> { children: Vec>>, /// The index into the proof vector that the encoding of this entry should be placed at. output_index: Option, - _marker: PhantomData, + is_inline: bool, + _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodec> StackEntry<'a, C> { +impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> + where + H: BinaryHasher, +{ fn new( prefix: LeftNibbleSlice<'a>, node_data: Vec, node_hash: Option, output_index: Option, + is_inline: bool, ) -> TrieResult { let node = OwnedNode::new::(node_data) @@ -69,12 +76,17 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { child_index: 0, children: vec![None; children_len], output_index, + is_inline, _marker: PhantomData::default(), }) } /// Encode this entry to an encoded trie node with data properly omitted. - fn encode_node(mut self) -> TrieResult, C::HashOut, C::Error> { + fn encode_node( + mut self, + complex: bool, + hash_buf: &mut H::Buffer, + ) -> TrieResult, C::HashOut, C::Error> { let node_data = self.node.data(); Ok(match self.node.node_plan() { NodePlan::Empty => node_data.to_vec(), @@ -103,12 +115,52 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { node_data, children, self.child_index, - &mut self.children + &mut self.children, )?; - C::branch_node( - self.children.into_iter(), - value_with_omission(node_data, value, self.omit_value) - ) + if !self.is_inline && complex { + let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); + let register_children = &mut register_children[..]; + let (mut result, no_child) = C::branch_node( + self.children.iter(), + value_with_omission(node_data, value, self.omit_value), + Some(register_children), + ); + no_child.trim_no_child(&mut result); + let bitmap_start = result.len(); + result.push(0u8); + result.push(0u8); + let mut in_proof_children = [false; NIBBLE_LENGTH]; + // write all inline nodes and ommitted node + // TODO again register for nothing + for (ix, child) in self.children.iter().enumerate() { + if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { + if *nb > 0 { + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + } + in_proof_children[ix] = true; + } + } + Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); + let additional_hashes = crate::trie_codec::binary_additional_hashes::( + &self.children[..], + &in_proof_children[..], + hash_buf, + ); + result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes + for hash in additional_hashes { + result.extend_from_slice(hash.as_ref()); + } + result + } else { + C::branch_node( + self.children.into_iter(), + value_with_omission(node_data, value, self.omit_value), + None, // TODO allow complex here + ).0 + } }, NodePlan::NibbledBranch { partial: partial_plan, value, children } => { let partial = partial_plan.build(node_data); @@ -118,12 +170,55 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { self.child_index, &mut self.children )?; - C::branch_node_nibbled( - partial.right_iter(), - partial.len(), - self.children.into_iter(), - value_with_omission(node_data, value, self.omit_value) - ) + if !self.is_inline && complex { + // TODO factor with non nibbled!! + let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); + let register_children = &mut register_children[..]; + let (mut result, no_child) = C::branch_node_nibbled( + partial.right_iter(), + partial.len(), + self.children.iter(), + value_with_omission(node_data, value, self.omit_value), + Some(register_children), + ); + no_child.trim_no_child(&mut result); + let bitmap_start = result.len(); + result.push(0u8); + result.push(0u8); + let mut in_proof_children = [false; NIBBLE_LENGTH]; + // write all inline nodes and ommitted node + // TODO again register for nothing + for (ix, child) in self.children.iter().enumerate() { + if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { + if *nb > 0 { + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + } + in_proof_children[ix] = true; + } + } + Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); + let additional_hashes = crate::trie_codec::binary_additional_hashes::( + &self.children[..], + &in_proof_children[..], + hash_buf, + ); + result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes + for hash in additional_hashes { + result.extend_from_slice(hash.as_ref()); + } + result + } else { + C::branch_node_nibbled( + partial.right_iter(), + partial.len(), + self.children.into_iter(), + value_with_omission(node_data, value, self.omit_value), + None, // TODO allow complex here + ).0 + } }, }) } @@ -166,7 +261,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { thus they are never descended into; \ thus set_child will not be called on an entry with one of these types" ), - NodePlan::Extension { child, .. } => { + NodePlan::Extension { child, .. } => { assert_eq!( self.child_index, 0, "extension nodes only have one child; \ @@ -232,10 +327,13 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) .collect::>(); keys.sort(); keys.dedup(); + let mut hash_buf = ::Buffer::default(); + let hash_buf = &mut hash_buf; + // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack = >>::new(); + let mut stack = >>::new(); // The mutated trie nodes comprising the final proof. let mut proof_nodes = Vec::new(); @@ -244,7 +342,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) let key = LeftNibbleSlice::new(key_bytes); // Unwind the stack until the new entry is a child of the last entry on the stack. - unwind_stack(&mut stack, &mut proof_nodes, Some(&key))?; + unwind_stack(&mut stack, &mut proof_nodes, Some(&key), L::COMPLEX_HASH, hash_buf)?; // Perform the trie lookup for the next key, recording the sequence of nodes traversed. let mut recorder = Recorder::new(); @@ -309,6 +407,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) child_record.data, Some(child_record.hash), Some(output_index), + false, )? } NodeHandle::Inline(data) => { @@ -322,6 +421,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) data.to_vec(), None, None, + true, )? } }; @@ -350,7 +450,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) } } - unwind_stack(&mut stack, &mut proof_nodes, None)?; + unwind_stack(&mut stack, &mut proof_nodes, None, L::COMPLEX_HASH, hash_buf)?; Ok(proof_nodes) } @@ -496,11 +596,15 @@ fn value_with_omission<'a>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( - stack: &mut Vec>, +fn unwind_stack( + stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, + complex: bool, + hash_buf: &mut H::Buffer, ) -> TrieResult<(), C::HashOut, C::Error> + where + H: BinaryHasher, { while let Some(entry) = stack.pop() { match maybe_key { @@ -512,7 +616,7 @@ fn unwind_stack( _ => { // Pop and finalize node from the stack. let index = entry.output_index; - let encoded = entry.encode_node()?; + let encoded = entry.encode_node(complex, hash_buf)?; if let Some(parent_entry) = stack.last_mut() { parent_entry.set_child(&encoded); } diff --git a/trie-db/src/proof/mod.rs b/trie-db/src/proof/mod.rs index 30f5206d..307704ee 100644 --- a/trie-db/src/proof/mod.rs +++ b/trie-db/src/proof/mod.rs @@ -197,7 +197,11 @@ mod tests { ); } - #[test] + // #[test] TODO this does not work for complex hash, restore + // when two test case. + // Reason it does not work is that the hash not in root are + // skipped so there is no extranous hash, error could be better + // still, but produce in advance children fn test_verify_extraneous_hash_reference() { let (root, proof, _) = test_generate_proof::( test_entries(), diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index eba4cb6a..afaac8af 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -14,12 +14,15 @@ use crate::rstd::{ convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec, vec::Vec, + iter::from_fn, iter::FromFn, }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{Node, NodeHandle}, NodeCodec, TrieHash, TrieLayout, + node::{Node, NodeHandle}, NodeCodec, TrieHash, TrieLayout, EncodedNoChild, }; use hash_db::Hasher; +use ordered_trie::{BinaryHasher, HasherComplex}; +use crate::node_codec::{Bitmap, BITMAP_LENGTH}; /// Errors that may occur during proof verification. Most of the errors types simply indicate that @@ -94,7 +97,7 @@ impl std::error::Error for } } -struct StackEntry<'a, C: NodeCodec> { +struct StackEntry<'a, C: NodeCodec, H> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, node: Node<'a>, @@ -106,42 +109,105 @@ struct StackEntry<'a, C: NodeCodec> { child_index: usize, /// The child references to use in reconstructing the trie nodes. children: Vec>>, - _marker: PhantomData, + complex: Option<(Bitmap, Vec)>, + _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodec> StackEntry<'a, C> { - fn new(node_data: &'a [u8], prefix: LeftNibbleSlice<'a>, is_inline: bool) +impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> + where + H: BinaryHasher, +{ + fn new(node_data: &'a [u8], prefix: LeftNibbleSlice<'a>, is_inline: bool, complex: bool) -> Result> { - let node = C::decode(node_data) - .map_err(Error::DecodeError)?; - let children_len = match node { - Node::Empty | Node::Leaf(..) => 0, - Node::Extension(..) => 1, - Node::Branch(..) | Node::NibbledBranch(..) => NIBBLE_LENGTH, + let children_len = NIBBLE_LENGTH; + let mut children = vec![None; NIBBLE_LENGTH]; // TODO use array + let (node, complex) = if !is_inline && complex { + // TODO factorize with trie_codec + let encoded_node = node_data; + let (mut node, mut offset) = C::decode_no_child(encoded_node) + .map_err(Error::DecodeError)?; + match &mut node { + Node::Branch(b_child, _) | Node::NibbledBranch(_, b_child, _) => { + if encoded_node.len() < offset + 3 { + // TODO new error or move this parte to codec trait and use codec error + return Err(Error::IncompleteProof); + } + let keys_position = Bitmap::decode(&encoded_node[offset..offset + BITMAP_LENGTH]); + offset += BITMAP_LENGTH; + + let mut nb_additional; + // inline nodes + loop { + let nb = encoded_node[offset] as usize; + offset += 1; + if nb >= 128 { + nb_additional = nb - 128; + break; + } + if encoded_node.len() < offset + nb + 2 { + return Err(Error::IncompleteProof); + } + let ix = encoded_node[offset] as usize; + offset += 1; + let inline = &encoded_node[offset..offset + nb]; + if ix >= NIBBLE_LENGTH { + return Err(Error::IncompleteProof); + } + b_child[ix] = Some(NodeHandle::Inline(inline)); + offset += nb; + } + let hash_len = ::NULL_HASH.len(); + let additional_len = nb_additional * hash_len; + if encoded_node.len() < offset + additional_len { + return Err(Error::IncompleteProof); + } + let additional_hashes = from_fn(move || { + if nb_additional > 0 { + let mut hash = ::default(); + hash.as_mut().copy_from_slice(&encoded_node[offset..offset + hash_len]); + offset += hash_len; + nb_additional -= 1; + Some(hash) + } else { + None + } + }); + // TODO dedicated iterator type instead of from_fn to avoid alloc + let additional_hashes: Vec = additional_hashes.collect(); + (node, Some((keys_position, additional_hashes))) + }, + _ => (node, None), + } + } else { + (C::decode(node_data) + .map_err(Error::DecodeError)?, None) }; let value = match node { Node::Empty | Node::Extension(_, _) => None, Node::Leaf(_, value) => Some(value), Node::Branch(_, value) | Node::NibbledBranch(_, _, value) => value, }; + + Ok(StackEntry { node, is_inline, prefix, value, child_index: 0, - children: vec![None; children_len], + children, + complex, _marker: PhantomData::default(), }) } /// Encode this entry to an encoded trie node with data properly reconstructed. - fn encode_node(mut self) -> Result, Error> { + fn encode_node(&mut self) -> Result<(Vec, EncodedNoChild), Error> { self.complete_children()?; Ok(match self.node { Node::Empty => - C::empty_node().to_vec(), + (C::empty_node().to_vec(), EncodedNoChild::Unused), Node::Leaf(partial, _) => { let value = self.value .expect( @@ -149,29 +215,37 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { value is only ever reassigned in the ValueMatch::MatchesLeaf match \ clause, which assigns only to Some" ); - C::leaf_node(partial.right(), value) + (C::leaf_node(partial.right(), value), EncodedNoChild::Unused) } Node::Extension(partial, _) => { let child = self.children[0] .expect("the child must be completed since child_index is 1"); - C::extension_node( + (C::extension_node( partial.right_iter(), partial.len(), child - ) + ), EncodedNoChild::Unused) } - Node::Branch(_, _) => + Node::Branch(_, _) => { + let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); + let register_children = &mut register_children[..]; C::branch_node( self.children.iter(), self.value, - ), - Node::NibbledBranch(partial, _, _) => + Some(register_children), // TODO again unused register result + ) + }, + Node::NibbledBranch(partial, _, _) => { + let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); + let register_children = &mut register_children[..]; C::branch_node_nibbled( partial.right_iter(), partial.len(), self.children.iter(), self.value, - ), + Some(register_children), // TODO again unused register result + ) + }, }) } @@ -179,6 +253,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { &mut self, child_prefix: LeftNibbleSlice<'a>, proof_iter: &mut I, + complex: bool, ) -> Result> where I: Iterator>, @@ -187,7 +262,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { Node::Extension(_, child) => { // Guaranteed because of sorted keys order. assert_eq!(self.child_index, 0); - Self::make_child_entry(proof_iter, child, child_prefix) + Self::make_child_entry(proof_iter, child, child_prefix, complex) } Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { // because this is a branch @@ -205,7 +280,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { } let child = children[self.child_index] .expect("guaranteed by advance_item"); - Self::make_child_entry(proof_iter, child, child_prefix) + Self::make_child_entry(proof_iter, child, child_prefix, complex) } _ => panic!("cannot have children"), } @@ -239,6 +314,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { proof_iter: &mut I, child: NodeHandle<'a>, prefix: LeftNibbleSlice<'a>, + complex: bool, ) -> Result> where I: Iterator>, @@ -248,9 +324,9 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { if data.is_empty() { let node_data = proof_iter.next() .ok_or(Error::IncompleteProof)?; - StackEntry::new(node_data, prefix, false) + StackEntry::new(node_data, prefix, false, complex) } else { - StackEntry::new(data, prefix, true) + StackEntry::new(data, prefix, true, complex) } } NodeHandle::Hash(data) => { @@ -423,7 +499,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve // A stack of child references to fill in omitted branch children for later trie nodes in the // proof. - let mut stack: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); let root_node = match proof_iter.next() { Some(node) => node, @@ -432,19 +508,24 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve let mut last_entry = StackEntry::new( root_node, LeftNibbleSlice::new(&[]), - false + false, + L::COMPLEX_HASH, )?; loop { // Insert omitted value. match last_entry.advance_item(&mut items_iter)? { Step::Descend(child_prefix) => { - let next_entry = last_entry.advance_child_index(child_prefix, &mut proof_iter)?; + let next_entry = last_entry.advance_child_index( + child_prefix, + &mut proof_iter, + L::COMPLEX_HASH, + )?; stack.push(last_entry); last_entry = next_entry; } Step::UnwindStack => { let is_inline = last_entry.is_inline; - let node_data = last_entry.encode_node()?; + let (node_data, no_child) = last_entry.encode_node()?; let child_ref = if is_inline { if node_data.len() > L::Hash::LENGTH { @@ -454,8 +535,41 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve &mut hash.as_mut()[..node_data.len()].copy_from_slice(node_data.as_ref()); ChildReference::Inline(hash, node_data.len()) } else { - let hash = L::Hash::hash(&node_data); - ChildReference::Hash(hash) + ChildReference::Hash(if let Some((bitmap_keys, additional_hash)) = last_entry.complex { + let children = last_entry.children; + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let children = children.into_iter() + .enumerate() + .filter_map(|(ix, v)| { + v.as_ref().map(|v| (ix, v.clone())) + }) + .map(|(ix, child_ref)| { + if bitmap_keys.value_at(ix) { + Some(match child_ref { + ChildReference::Hash(h) => h, + ChildReference::Inline(h, _) => h, + }) + } else { + None + } + }); + + if let Some(h) = L::Hash::hash_complex( + &no_child.encoded_no_child(node_data.as_slice())[..], + nb_children, + children, + additional_hash.into_iter(), + true, + ) { + h + } else { + // TODO better error for the invalid + // complex hash + return Err(Error::RootMismatch(Default::default())); + } + } else { + L::Hash::hash(&node_data) + }) }; if let Some(entry) = stack.pop() { @@ -482,4 +596,4 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } Ok(()) -} \ No newline at end of file +} diff --git a/trie-db/src/recorder.rs b/trie-db/src/recorder.rs index 90057c05..68d571cf 100644 --- a/trie-db/src/recorder.rs +++ b/trie-db/src/recorder.rs @@ -132,7 +132,7 @@ mod tests { }); } - #[test] + // #[test] TODO put it back for reftriedb not complex fn trie_record() { let mut db = MemoryDB::, _>::default(); let mut root = Default::default(); diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs index 06edf03e..aeff2cef 100644 --- a/trie-db/src/sectriedbmut.rs +++ b/trie-db/src/sectriedbmut.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{HashDB, Hasher}; +use hash_db::{Hasher}; +use crate::node_codec::HashDBComplexDyn as HashDB; use super::{Result, DBValue, TrieMut, TrieDBMut, TrieLayout, TrieHash, CError}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 7d4783f8..e0ee8ed2 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -25,16 +25,19 @@ //! expected to save roughly (n - 1) hashes in size where n is the number of nodes in the partial //! trie. -use hash_db::HashDB; +pub use ordered_trie::{BinaryHasher, HashDBComplex}; +use hash_db::{HashDB, Hasher}; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; +use crate::node_codec::{Bitmap, BITMAP_LENGTH, EncodedNoChild}; use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, rc::Rc, result, vec, vec::Vec, + iter::from_fn, ops::Range, }; - +use ordered_trie::{SequenceBinaryTree, HashProof, trie_root, UsizeKeyNode}; struct EncoderStackEntry { /// The prefix is the nibble path to the node in the trie. prefix: NibbleVec, @@ -43,7 +46,7 @@ struct EncoderStackEntry { /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// Flags indicating whether each child is omitted in the encoded node. - omit_children: Vec, + omit_children: [bool; NIBBLE_LENGTH], /// The encoding of the subtrie nodes rooted at this entry, which is built up in /// `encode_compact`. output_index: usize, @@ -95,7 +98,14 @@ impl EncoderStackEntry { } /// Generates the encoding of the subtrie rooted at this entry. - fn encode_node(&self) -> Result, C::HashOut, C::Error> { + fn encode_node( + &self, + complex_hash: bool, + hash_buf: &mut H::Buffer, + ) -> Result, C::HashOut, C::Error> + where + H: BinaryHasher, + { let node_data = self.node.data(); Ok(match self.node.node_plan() { NodePlan::Empty | NodePlan::Leaf { .. } => node_data.to_vec(), @@ -109,19 +119,94 @@ impl EncoderStackEntry { } } NodePlan::Branch { value, children } => { - C::branch_node( - Self::branch_children(node_data, &children, &self.omit_children)?.iter(), - value.clone().map(|range| &node_data[range]) - ) + let mut register: [Option<_>; NIBBLE_LENGTH]; // TODO unused register + let (children, complex) = if complex_hash { + let no_omit = [false; NIBBLE_LENGTH]; + register = Default::default(); + (Self::branch_children(node_data, &children, &no_omit[..])?, Some(&mut register[..])) + } else { + (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) + }; + let (mut result, no_child) = C::branch_node( + children.iter(), + value.clone().map(|range| &node_data[range]), + complex, + ); + if complex_hash { + no_child.trim_no_child(&mut result); + let bitmap_start = result.len(); + result.push(0u8); + result.push(0u8); + let mut in_proof_children = self.omit_children.clone(); + // write all inline nodes TODO we could omit children first + // as in std case and fill this bitmap as in generate.rs. + for (ix, child) in children.iter().enumerate() { + if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + in_proof_children[ix] = true; + } + } + Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); + let additional_hashes = binary_additional_hashes::( + &children[..], + &in_proof_children[..], + hash_buf, + ); + result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes + for hash in additional_hashes { + result.extend_from_slice(hash.as_ref()); + } + } + result } NodePlan::NibbledBranch { partial, value, children } => { + let mut register: [Option<_>; NIBBLE_LENGTH]; // TODO unused register + let (children, complex) = if complex_hash { + let no_omit = [false; NIBBLE_LENGTH]; + register = Default::default(); + (Self::branch_children(node_data, &children, &no_omit[..])?, Some(&mut register[..])) + } else { + (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) + }; let partial = partial.build(node_data); - C::branch_node_nibbled( + let (mut result, no_child) = C::branch_node_nibbled( partial.right_iter(), partial.len(), - Self::branch_children(node_data, &children, &self.omit_children)?.iter(), - value.clone().map(|range| &node_data[range]) - ) + children.iter(), + value.clone().map(|range| &node_data[range]), + complex, + ); + if complex_hash { + no_child.trim_no_child(&mut result); + let bitmap_start = result.len(); + result.push(0u8); + result.push(0u8); + let mut in_proof_children = self.omit_children.clone(); + // write all inline nodes + for (ix, child) in children.iter().enumerate() { + if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + in_proof_children[ix] = true; + } + } + Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); + let additional_hashes = binary_additional_hashes::( + &children[..], + &in_proof_children[..], + hash_buf, + ); + result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes + for hash in additional_hashes { + result.extend_from_slice(hash.as_ref()); + } + } + result } }) } @@ -171,6 +256,10 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE { let mut output = Vec::new(); + // TODO make it optional and replace boolean is_complex + let mut hash_buf = ::Buffer::default(); + let hash_buf = &mut hash_buf; + // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. let mut stack: Vec> = Vec::new(); @@ -212,20 +301,18 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE stack.push(last_entry); break; } else { - output[last_entry.output_index] = last_entry.encode_node()?; + output[last_entry.output_index] = last_entry.encode_node::( + L::COMPLEX_HASH, + hash_buf, + )?; } } - let children_len = match node.node_plan() { - NodePlan::Empty | NodePlan::Leaf { .. } => 0, - NodePlan::Extension { .. } => 1, - NodePlan::Branch { .. } | NodePlan::NibbledBranch { .. } => NIBBLE_LENGTH, - }; stack.push(EncoderStackEntry { prefix, node, child_index: 0, - omit_children: vec![false; children_len], + omit_children: [false; NIBBLE_LENGTH], output_index: output.len(), _marker: PhantomData::default(), }); @@ -244,23 +331,29 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE } while let Some(entry) = stack.pop() { - output[entry.output_index] = entry.encode_node()?; + output[entry.output_index] = entry.encode_node::( + L::COMPLEX_HASH, + hash_buf, + )?; } Ok(output) } -struct DecoderStackEntry<'a, C: NodeCodec> { +struct DecoderStackEntry<'a, C: NodeCodec, F> { node: Node<'a>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The reconstructed child references. + /// TODO remove Vec here!!! children: Vec>>, + /// Complex proof input + complex: Option<(Bitmap, F)>, _marker: PhantomData, } -impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { +impl<'a, C: NodeCodec, F> DecoderStackEntry<'a, C, F> { /// Advance the child index until either it exceeds the number of children or the child is /// marked as omitted. Omitted children are indicated by an empty inline reference. For each /// child that is passed over and not omitted, copy over the child reference from the node to @@ -286,20 +379,39 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { self.child_index += 1; } Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { - while self.child_index < NIBBLE_LENGTH { - match children[self.child_index] { - Some(NodeHandle::Inline(data)) if data.is_empty() => - return Ok(false), - Some(child) => { - let child_ref = child.try_into() - .map_err(|hash| Box::new( - TrieError::InvalidHash(C::HashOut::default(), hash) - ))?; - self.children[self.child_index] = Some(child_ref); + if let Some((bitmap, _)) = self.complex.as_ref() { + while self.child_index < NIBBLE_LENGTH { + match children[self.child_index] { + Some(NodeHandle::Inline(data)) if data.is_empty() && bitmap.value_at(self.child_index) => { + return Ok(false); + }, + Some(child) => { + let child_ref = child.try_into() + .map_err(|hash| Box::new( + TrieError::InvalidHash(C::HashOut::default(), hash) + ))?; + self.children[self.child_index] = Some(child_ref); + }, + None => {}, + } + self.child_index += 1; + } + } else { + while self.child_index < NIBBLE_LENGTH { + match children[self.child_index] { + Some(NodeHandle::Inline(data)) if data.is_empty() => + return Ok(false), + Some(child) => { + let child_ref = child.try_into() + .map_err(|hash| Box::new( + TrieError::InvalidHash(C::HashOut::default(), hash) + ))?; + self.children[self.child_index] = Some(child_ref); + } + None => {} } - None => {} + self.child_index += 1; } - self.child_index += 1; } } _ => {} @@ -347,27 +459,32 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { /// /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. - fn encode_node(self) -> Vec { + fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, EncodedNoChild) { match self.node { Node::Empty => - C::empty_node().to_vec(), + (C::empty_node().to_vec(), EncodedNoChild::Unused), Node::Leaf(partial, value) => - C::leaf_node(partial.right(), value), + (C::leaf_node(partial.right(), value), EncodedNoChild::Unused), Node::Extension(partial, _) => - C::extension_node( + (C::extension_node( partial.right_iter(), partial.len(), self.children[0] .expect("required by method precondition; qed"), - ), + ), EncodedNoChild::Unused), Node::Branch(_, value) => - C::branch_node(self.children.into_iter(), value), + C::branch_node( + self.children.into_iter(), + value, + register_children, + ), Node::NibbledBranch(partial, _, value) => C::branch_node_nibbled( partial.right_iter(), partial.len(), self.children.iter(), value, + register_children, ), } } @@ -389,19 +506,81 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDB, + DB: HashDBComplex, { // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); // The prefix of the next item to be read from the slice of encoded items. let mut prefix = NibbleVec::new(); for (i, encoded_node) in encoded.iter().enumerate() { - let node = L::Codec::decode(encoded_node) - .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))?; - + let (node, complex) = if L::COMPLEX_HASH { + let (mut node, mut offset) = L::Codec::decode_no_child(encoded_node) + .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))?; + match &mut node { + Node::Branch(b_child, _) | Node::NibbledBranch(_, b_child, _) => { + if encoded_node.len() < offset + 3 { + // TODO new error or move this parte to codec trait and use codec error + return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); + } + let keys_position = Bitmap::decode(&encoded_node[offset..offset + BITMAP_LENGTH]); + offset += BITMAP_LENGTH; + + let mut nb_additional; + // inline nodes + loop { + let nb = encoded_node[offset] as usize; + offset += 1; + if nb >= 128 { + nb_additional = nb - 128; + break; + } + if encoded_node.len() < offset + nb + 2 { + // TODO new error or move this parte to codec trait and use codec error + return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); + } + let ix = encoded_node[offset] as usize; + offset += 1; + let inline = &encoded_node[offset..offset + nb]; + if ix >= NIBBLE_LENGTH { + // TODO new error or move this parte to codec trait and use codec error + return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); + } + b_child[ix] = Some(NodeHandle::Inline(inline)); + offset += nb; + } + let hash_len = ::NULL_HASH.len(); + let additional_len = nb_additional * hash_len; + if encoded_node.len() < offset + additional_len { + // TODO new error or move this parte to codec trait and use codec error + return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); + } + let additional_hashes = from_fn(move || { + if nb_additional > 0 { + let mut hash = >::default(); + hash.as_mut().copy_from_slice(&encoded_node[offset..offset + hash_len]); + offset += hash_len; + nb_additional -= 1; + Some(hash) + } else { + None + } + }); + (node, Some((keys_position, additional_hashes))) + }, + _ => { + (node, None) + }, + } + } else { + ( + L::Codec::decode(encoded_node) + .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))?, + None, + ) + }; let children_len = match node { Node::Empty | Node::Leaf(..) => 0, Node::Extension(..) => 1, @@ -411,6 +590,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) node, child_index: 0, children: vec![None; children_len], + complex, _marker: PhantomData::default(), }; @@ -421,10 +601,47 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) break; } + let mut register_children: [Option<_>; NIBBLE_LENGTH]; + let mut register_children = if last_entry.complex.is_some() { + register_children = Default::default(); + Some(&mut register_children[..]) + } else { + None + }; + let complex = last_entry.complex.take(); // Since `advance_child_index` returned true, the preconditions for `encode_node` are // satisfied. - let node_data = last_entry.encode_node(); - let node_hash = db.insert(prefix.as_prefix(), node_data.as_ref()); + let (node_data, no_child) = last_entry.encode_node(register_children.as_mut().map(|r| r.as_mut())); + let node_hash = if let Some((bitmap_keys, additional_hashes)) = complex { + let children = register_children.expect("Set to some if complex"); + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let children = children.iter() + .enumerate() + .filter_map(|(ix, v)| { + v.as_ref().map(|v| (ix, v.clone())) + }) + .map(|(ix, range)| { + if bitmap_keys.value_at(ix) { + let mut v = TrieHash::::default(); + let len = range.end - range.start; + v.as_mut()[..len].copy_from_slice(&node_data[range]); + Some(v) + } else { + None + } + }); + db.insert_complex( + prefix.as_prefix(), + &node_data[..], + no_child.encoded_no_child(&node_data[..]), + nb_children, + children, + additional_hashes, + true, + ) + } else { + db.insert(prefix.as_prefix(), node_data.as_ref()) + }; if let Some(entry) = stack.pop() { last_entry = entry; @@ -441,6 +658,37 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) Err(Box::new(TrieError::IncompleteDatabase(>::default()))) } +pub(crate) fn binary_additional_hashes( + children: &[Option>], + in_proof_children: &[bool], + hash_buf: &mut H::Buffer, +) -> Vec { + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let tree = SequenceBinaryTree::new(0, 0, nb_children); + + let to_prove = children.iter().zip(in_proof_children.iter()) + .filter_map(|(o_child, in_proof)| o_child.as_ref().map(|_| *in_proof)) + // correct iteration over binary tree + .zip(tree.iter_path_node_key::(None)) + .filter_map(|(in_proof, ix_key)| if in_proof { + Some(ix_key) + } else { + None + }); + + + let mut callback = HashProof::::new(hash_buf, to_prove); + let hashes = children.iter() + .filter_map(|o_child| o_child.as_ref()) + .map(|child| match child { + ChildReference::Hash(h) => h.clone(), + ChildReference::Inline(h, _) => h.clone(), + }); + // TODO we can skip a hash (the one to calculate root) + let _root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); + callback.take_additional_hash() +} + #[cfg(test)] mod tests { use crate::DBValue; @@ -486,7 +734,7 @@ mod tests { // Populate a partial trie DB with recorded nodes. let mut partial_db = MemoryDB::default(); for record in recorder.drain() { - partial_db.insert(EMPTY_PREFIX, &record.data); + partial_db.emplace(record.hash, EMPTY_PREFIX, record.data); } // Compactly encode the partial trie DB. diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index fb29eb40..b07d421b 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -19,13 +19,15 @@ use super::{Result, TrieError, TrieMut, TrieLayout, TrieHash, CError}; use super::lookup::Lookup; use super::node::{NodeHandle as EncodedNodeHandle, Node as EncodedNode, decode_hash}; +use crate::node_codec::HashDBComplexDyn; use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::node_codec::NodeCodec; +use crate::node_codec::{NodeCodec, EncodedNoChild}; use crate::nibble::{NibbleVec, NibbleSlice, nibble_ops, BackingByteVec}; use crate::rstd::{ boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, + ops::Range, }; #[cfg(feature = "std")] @@ -124,7 +126,7 @@ where fn inline_or_hash( parent_hash: H::Out, child: EncodedNodeHandle, - db: &dyn HashDB, + db: &dyn HashDBComplexDyn, storage: &mut NodeStorage ) -> Result, H::Out, C::Error> where @@ -149,7 +151,7 @@ where fn from_encoded<'a, 'b, C, H>( node_hash: H::Out, data: &'a[u8], - db: &dyn HashDB, + db: &dyn HashDBComplexDyn, storage: &'b mut NodeStorage, ) -> Result where @@ -203,27 +205,30 @@ where } // TODO: parallelize - fn into_encoded(self, mut child_cb: F) -> Vec - where + fn into_encoded( + self, + mut child_cb: F, + register_children: Option<&mut [Option>]>, + ) -> (Vec, EncodedNoChild) where C: NodeCodec, F: FnMut(NodeHandle, Option<&NibbleSlice>, Option) -> ChildReference, H: Hasher, { match self { - Node::Empty => C::empty_node().to_vec(), + Node::Empty => (C::empty_node().to_vec(), EncodedNoChild::Unused), Node::Leaf(partial, value) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - C::leaf_node(pr.right(), &value) + (C::leaf_node(pr.right(), &value), EncodedNoChild::Unused) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let it = pr.right_iter(); let c = child_cb(child, Some(&pr), None); - C::extension_node( + (C::extension_node( it, pr.len(), c, - ) + ), EncodedNoChild::Unused) }, Node::Branch(mut children, value) => { C::branch_node( @@ -234,7 +239,8 @@ where .map(|(i, maybe_child)| { maybe_child.map(|child| child_cb(child, None, Some(i as u8))) }), - value.as_ref().map(|v| &v[..]) + value.as_ref().map(|v| &v[..]), + register_children, ) }, Node::NibbledBranch(partial, mut children, value) => { @@ -254,7 +260,8 @@ where child_cb(child, Some(&pr), Some(i as u8)) }) }), - value.as_ref().map(|v| &v[..]) + value.as_ref().map(|v| &v[..]), + register_children, ) }, } @@ -384,7 +391,7 @@ impl<'a, H> Index<&'a StorageHandle> for NodeStorage { } } -/// A `Trie` implementation using a generic `HashDB` backing database. +/// A `Trie` implementation using a generic `HashDBComplexDyn` backing database. /// /// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. /// Note that changes are not committed to the database until `commit` is called. @@ -416,7 +423,7 @@ where L: TrieLayout, { storage: NodeStorage>, - db: &'a mut dyn HashDB, + db: &'a mut dyn HashDBComplexDyn, root: &'a mut TrieHash, root_handle: NodeHandle>, death_row: HashSet<(TrieHash, (BackingByteVec, Option))>, @@ -429,8 +436,20 @@ impl<'a, L> TrieDBMut<'a, L> where L: TrieLayout, { + #[inline] + fn register_children_buf(node: &Node>) -> Option<[Option>; 16]> { + match node { + Node::NibbledBranch(..) | Node::Branch(..) => if L::COMPLEX_HASH { + Some(Default::default()) + } else { + None + }, + _ => None, + } + + } /// Create a new trie with backing database `db` and empty `root`. - pub fn new(db: &'a mut dyn HashDB, root: &'a mut TrieHash) -> Self { + pub fn new(db: &'a mut dyn HashDBComplexDyn, root: &'a mut TrieHash) -> Self { *root = L::Codec::hashed_null_node(); let root_handle = NodeHandle::Hash(L::Codec::hashed_null_node()); @@ -447,7 +466,7 @@ where /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. pub fn from_existing( - db: &'a mut dyn HashDB, + db: &'a mut dyn HashDBComplexDyn, root: &'a mut TrieHash, ) -> Result, CError> { if !db.contains(root, EMPTY_PREFIX) { @@ -465,12 +484,12 @@ where }) } /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { + pub fn db(&self) -> &dyn HashDBComplexDyn { self.db } /// Get the backing database mutably. - pub fn db_mut(&mut self) -> &mut dyn HashDB { + pub fn db_mut(&mut self) -> &mut dyn HashDBComplexDyn { self.db } @@ -1421,17 +1440,28 @@ where match self.storage.destroy(handle) { Stored::New(node) => { let mut k = NibbleVec::new(); - let encoded_root = node.into_encoded::<_, L::Codec, L::Hash>( + let mut register_children = Self::register_children_buf(&node); + let (encoded_root, no_node) = node.into_encoded::<_, L::Codec, L::Hash>( |child, o_slice, o_index| { let mov = k.append_optional_slice_and_nibble(o_slice, o_index); let cr = self.commit_child(child, &mut k); k.drop_lasts(mov); cr - } + }, + register_children.as_mut().map(|a| &mut a[..]), ); #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:#x?}", &encoded_root[..]); - *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root[..]); + if let Some(children) = register_children { + *self.root = self.db.insert_complex( + EMPTY_PREFIX, + &encoded_root[..], + &children[..], + no_node, + ); + } else { + *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root[..]); + } self.hash_count += 1; self.root_handle = NodeHandle::Hash(*self.root); @@ -1462,7 +1492,8 @@ where match self.storage.destroy(storage_handle) { Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { - let encoded = { + let mut register_children = Self::register_children_buf(&node); + let (encoded, no_node) = { let commit_child = | node_handle, o_slice: Option<&NibbleSlice>, @@ -1473,10 +1504,23 @@ where prefix.drop_lasts(mov); cr }; - node.into_encoded::<_, L::Codec, L::Hash>(commit_child) + node.into_encoded::<_, L::Codec, L::Hash>( + commit_child, + register_children.as_mut().map(|a| &mut a[..]), + ) }; if encoded.len() >= L::Hash::LENGTH { - let hash = self.db.insert(prefix.as_prefix(), &encoded[..]); + let hash = if let Some(children) = register_children { + self.db.insert_complex( + prefix.as_prefix(), + &encoded[..], + &children[..], + no_node, + ) + } else { + self.db.insert(prefix.as_prefix(), &encoded[..]) + }; + self.hash_count +=1; ChildReference::Hash(hash) } else { @@ -1613,14 +1657,15 @@ mod tests { use log::debug; use crate::DBValue; use memory_db::{MemoryDB, PrefixedKey}; - use hash_db::{Hasher, HashDB}; + use hash_db::Hasher; use keccak_hasher::KeccakHasher; - use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, - ReferenceNodeCodec, reference_trie_root, reference_trie_root_no_extension}; + use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, HashDBComplexDyn, + ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, + reference_trie_root_no_extension_iter_build as reference_trie_root_no_extension}; use crate::nibble::BackingByteVec; fn populate_trie<'db>( - db: &'db mut dyn HashDB, + db: &'db mut dyn HashDBComplexDyn, root: &'db mut ::Out, v: &[(Vec, Vec)] ) -> RefTrieDBMut<'db> { @@ -1641,7 +1686,7 @@ mod tests { } fn populate_trie_no_extension<'db>( - db: &'db mut dyn HashDB, + db: &'db mut dyn HashDBComplexDyn, root: &'db mut ::Out, v: &[(Vec, Vec)] ) -> RefTrieDBMutNoExt<'db> { From 7ae42ffc00a14b93ad3e157964147d2162d03558 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Apr 2020 10:54:50 +0200 Subject: [PATCH 02/73] remove warnings --- memory-db/src/lib.rs | 5 ++- ordered-trie/src/lib.rs | 45 +++++++++++++++----------- test-support/reference-trie/src/lib.rs | 1 - trie-db/src/iter_build.rs | 14 +++++--- trie-db/src/lib.rs | 17 ++-------- trie-db/src/node_codec.rs | 36 ++++++++++++++++++--- trie-db/src/proof/verify.rs | 5 ++- trie-db/src/trie_codec.rs | 1 - trie-db/src/triedbmut.rs | 2 +- 9 files changed, 74 insertions(+), 52 deletions(-) diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index a08ab0f8..c8db4c9f 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -19,9 +19,9 @@ #[cfg(not(feature = "std"))] extern crate alloc; -use ordered_trie::{HashOnly, HashDBComplex, HasherComplex, BinaryHasher}; +use ordered_trie::{HashDBComplex, HasherComplex}; use hash_db::{HashDB, HashDBRef, PlainDB, PlainDBRef, Hasher as KeyHasher, - AsHashDB, AsPlainDB, Prefix, ComplexLayout}; + AsHashDB, AsPlainDB, Prefix}; use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; #[cfg(feature = "deprecated")] #[cfg(feature = "std")] @@ -35,7 +35,6 @@ use std::{ marker::PhantomData, cmp::Eq, borrow::Borrow, - ops::Range, }; #[cfg(not(feature = "std"))] diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 36792272..fb27ae2e 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -38,14 +38,11 @@ mod rstd { impl Error for T {} } -#[cfg(feature = "std")] -use self::rstd::{fmt, Error}; - -use hash_db::{MaybeDebug, AsHashDB, Prefix, HashDB}; -use self::rstd::{boxed::Box, vec::Vec}; +use hash_db::{Prefix, HashDB}; +use self::rstd::vec::Vec; -use hash_db::{HashDBRef, Hasher}; +use hash_db::Hasher; use crate::rstd::marker::PhantomData; @@ -139,6 +136,7 @@ pub struct SequenceBinaryTree { _ph: PhantomData, } +/* TODO unimplemented pub struct SequenceBinaryTreeDB<'a, K, H: Hasher> { tree: &'a SequenceBinaryTree, db: &'a dyn HashDBRef, @@ -149,6 +147,7 @@ pub struct SequenceBinaryTreeInMem<'a, K, NK: Ord, H: Hasher> { tree: &'a SequenceBinaryTree, db: &'a crate::rstd::BTreeMap, } +*/ impl Default for SequenceBinaryTree { fn default() -> Self { @@ -240,6 +239,7 @@ impl SequenceBinaryTree { self.length - self.start - self.end } + #[cfg(test)] // TODO is in implementation but unused, should be rename to resize fn push(&mut self, mut nb: usize) { if nb == 0 { return; @@ -311,7 +311,7 @@ impl SequenceBinaryTree { } pub fn iter_depth(&self, from: Option) -> impl Iterator { - if let Some(from) = from { + if let Some(_from) = from { unimplemented!(); } let nb_elements = self.nb_elements(); @@ -348,7 +348,7 @@ impl SequenceBinaryTree { where KN: KeyNode + From<(usize, usize)> + Clone, { - if let Some(from) = from { + if let Some(_from) = from { unimplemented!(); } let nb_elements = self.nb_elements(); @@ -385,8 +385,8 @@ impl SequenceBinaryTree { } }) } - - fn pop(&mut self, nb: usize) { +/* TODO unimplemented + fn pop(&mut self, _nb: usize) { unimplemented!("update max depth"); } @@ -395,18 +395,19 @@ impl SequenceBinaryTree { // TODO if start = max_depth_length / 2 -> max_depth - 1 } - fn max_depth_length(end: &usize) -> usize { + fn max_depth_length(_end: &usize) -> usize { // 2^x = max_depth_length unimplemented!() } - fn front_depth(index: usize) -> usize { + fn front_depth(_index: usize) -> usize { unimplemented!("for index between end and max len"); } - fn tail_depth(index: usize) -> usize { + fn tail_depth(_index: usize) -> usize { unimplemented!("for index between end and max len"); } +*/ } @@ -617,7 +618,7 @@ impl KeyNode for UsizeKeyNode { } fn push_back(&mut self, nibble: bool) { self.value = self.value << 1; - self.value = self.value | 1; + self.value = self.value | (nibble as usize); self.depth +=1; } fn pop_front(&mut self) -> Option { @@ -633,7 +634,7 @@ impl KeyNode for UsizeKeyNode { } fn push_front(&mut self, nibble: bool) { self.depth += 1; - self.value = self.value | (1 << (self.depth - 1)); + self.value = self.value | ((nibble as usize) << (self.depth - 1)); } fn remove_at(&mut self, index: usize) { @@ -700,8 +701,14 @@ fn key_node_test() { assert_eq!(u.push_back(true), v.push_back(true)); } assert_eq!(u.pop_back(), v.pop_back()); + if !end { + assert_eq!(u.push_back(false), v.push_back(false)); + } + assert_eq!(u.pop_back(), v.pop_back()); assert_eq!(u.push_front(true), v.push_front(true)); assert_eq!(u.pop_front(), v.pop_front()); + assert_eq!(u.push_front(false), v.push_front(false)); + assert_eq!(u.pop_front(), v.pop_front()); if !end { assert_eq!(start, u.into()); assert_eq!(start, v.clone().into()); @@ -764,7 +771,7 @@ pub trait ProcessNodeProof: ProcessNode { } /// Does only proccess hash on its buffer. -/// Buffer length need to be right and is unchecked. +/// Correct buffer size is expected, this is unchecked. pub struct HashOnly<'a, H>(&'a mut [u8], PhantomData); impl<'a, H: BinaryHasher> HashOnly<'a, H> { @@ -1000,7 +1007,7 @@ pub fn trie_root(layout: &SequenceBinaryTree, input: I, cal F: ProcessNode, { debug_assert!(layout.start == 0, "unimplemented start"); - let mut iter = input.into_iter().zip(layout.iter_path_node_key::(None)).enumerate(); + let mut iter = input.into_iter().zip(layout.iter_path_node_key::(None)); debug_assert!({ let (r, s) = iter.size_hint(); if s == Some(r) { @@ -1010,7 +1017,7 @@ pub fn trie_root(layout: &SequenceBinaryTree, input: I, cal } }); let mut depth1 = layout.depth; - let mut child1 = if let Some((_, (child, key))) = iter.next() { + let mut child1 = if let Some((child, key)) = iter.next() { debug_assert!(key.depth() == depth1); child } else { @@ -1034,7 +1041,7 @@ pub fn trie_root(layout: &SequenceBinaryTree, input: I, cal depth1 = key.depth(); child1 = parent; } else { - if let Some((index, (child2, key2))) = iter.next() { + if let Some((child2, key2)) = iter.next() { key = key2; if key.depth() == depth1 { key.pop_back(); diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index a570f682..2059ca90 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -28,7 +28,6 @@ use trie_db::{ trie_visit, TrieBuilderComplex, TrieRootComplex, - TrieRootUnhashedComplex, Partial, BinaryHasher, EncodedNoChild, diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index af72aaf6..9908a6c4 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -17,7 +17,7 @@ //! implementation. //! See `trie_visit` function. -use hash_db::{Hasher, HashDB, Prefix, ComplexLayout}; +use hash_db::{Hasher, HashDB, Prefix}; use ordered_trie::{HashDBComplex, HasherComplex}; use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range}; use crate::triedbmut::{ChildReference}; @@ -401,7 +401,8 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> prefix: Prefix, (encoded_node, _no_child): (Vec, EncodedNoChild), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + // TODO different trait?? + _complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -480,7 +481,8 @@ impl ProcessEncodedNode<::Out> for TrieRoot, EncodedNoChild), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + // TODO different trait + _complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -600,7 +602,8 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, EncodedNoChild), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + // TODO different trait? + _complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { println!("Encoded node: {:x?}", &encoded_node); println!(" with prefix: {:x?}", &p); @@ -627,7 +630,8 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { _: Prefix, (encoded_node, _no_child): (Vec, EncodedNoChild), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + // TODO different trait + _complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 1d8f5d34..e47214d5 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -38,7 +38,7 @@ mod rstd { } #[cfg(feature = "std")] -use self::rstd::{fmt, Error}; +use self::rstd::fmt; use hash_db::MaybeDebug; use self::rstd::{boxed::Box, vec::Vec}; @@ -70,7 +70,7 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, Partial, HashDBComplexDyn, EncodedNoChild, Bitmap, BITMAP_LENGTH}; +pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, EncodedNoChild, Bitmap, BITMAP_LENGTH}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; pub use crate::iterator::TrieDBNodeIterator; @@ -125,19 +125,6 @@ impl fmt::Display for TrieError where T: MaybeDebug, E: MaybeDebug { } } -#[cfg(feature = "std")] -impl Error for TrieError where T: fmt::Debug, E: Error { - fn description(&self) -> &str { - match *self { - TrieError::InvalidStateRoot(_) => "Invalid state root", - TrieError::IncompleteDatabase(_) => "Incomplete database", - TrieError::ValueAtIncompleteKey(_, _) => "Value at incomplete key", - TrieError::DecoderError(_, ref err) => err.description(), - TrieError::InvalidHash(_, _) => "Encoded node contains invalid hash reference", - } - } -} - /// Trie result type. /// Boxed to avoid copying around extra space for the `Hasher`s `Out` on successful queries. pub type Result = crate::rstd::result::Result>>; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index e6a514d2..b67f85a6 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -97,6 +97,37 @@ pub trait NodeCodec: Sized { ) -> (Vec, EncodedNoChild); } +/// Positional input for children decoding. +pub type OffsetChildren = usize; + +/// Trait for handling complex proof. +/// This adds methods to basic node codec in order to support: +/// - storage encoding with existing `NodeCodec` methods +/// - encode a proof specific representation. (usually the common representation and +/// the merkle proof of the children stored encoded hash). +/// - Intermediate optional common representation shared between storage +pub trait NodeCodecComplex: NodeCodec { + /// Sequence of hashes needed for the children proof verification. + type AdditionalHashes: Iterator; + + /// `Decode_no_child`, returning an offset position if there is a common representation. + /// NodePlan do not include child (sized null). + /// TODO EMCH this is technical function for common implementation. + fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, OffsetChildren), Self::Error>; + + /// Decode but child are not included (instead we put empty inline + /// nodes). + /// An children positianal information is also return for decoding of children. + /// TODO EMCH this looks rather useless. + fn decode_node_proof(data: &[u8]) -> Result<(Node, OffsetChildren), Self::Error> { + let (plan, offset) = Self::decode_plan_proof(data)?; + Ok((plan.build(data), offset)) + } + + fn decode_proof(data: &[u8]) -> Result<(NodePlan, Self::AdditionalHashes), Self::Error>; + +} + #[derive(Clone)] pub enum EncodedNoChild { // not runing complex @@ -166,7 +197,6 @@ impl> HashDBComplexDyn for C { // TODO factor this with iter_build (just use the trait) let nb_children = children.iter().filter(|v| v.is_some()).count(); let children = ComplexLayoutIterValues::new( - nb_children, children.iter().filter_map(|v| v.as_ref()), value, ); @@ -207,7 +237,6 @@ impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDBComplexDyn { // TODO this using a buffer is bad (we should switch // binary hasher to use slice as input (or be able to)) pub struct ComplexLayoutIterValues<'a, HO, I> { - nb_children: usize, children: I, node: &'a [u8], _ph: PhantomData, @@ -230,9 +259,8 @@ code snippet for proof */ impl<'a, HO: Default, I> ComplexLayoutIterValues<'a, HO, I> { - pub fn new(nb_children: usize, children: I, node: &'a[u8]) -> Self { + pub fn new(children: I, node: &'a[u8]) -> Self { ComplexLayoutIterValues { - nb_children, children, node, _ph: PhantomData, diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index afaac8af..deac9439 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -14,7 +14,7 @@ use crate::rstd::{ convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec, vec::Vec, - iter::from_fn, iter::FromFn, + iter::from_fn, }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, @@ -120,8 +120,7 @@ impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> fn new(node_data: &'a [u8], prefix: LeftNibbleSlice<'a>, is_inline: bool, complex: bool) -> Result> { - let children_len = NIBBLE_LENGTH; - let mut children = vec![None; NIBBLE_LENGTH]; // TODO use array + let children = vec![None; NIBBLE_LENGTH]; // TODO use array let (node, complex) = if !is_inline && complex { // TODO factorize with trie_codec let encoded_node = node_data; diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index e0ee8ed2..fc7b8d78 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -26,7 +26,6 @@ //! trie. pub use ordered_trie::{BinaryHasher, HashDBComplex}; -use hash_db::{HashDB, Hasher}; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index b07d421b..a273763e 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -20,7 +20,7 @@ use super::lookup::Lookup; use super::node::{NodeHandle as EncodedNodeHandle, Node as EncodedNode, decode_hash}; use crate::node_codec::HashDBComplexDyn; -use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; +use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; use crate::node_codec::{NodeCodec, EncodedNoChild}; From 86dae183e1b9f5711f98e1f9887061cda36cd8cf Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Apr 2020 12:12:00 +0200 Subject: [PATCH 03/73] use a hashes plan for additional hash, this is an iterator, no need to buff plans. --- ordered-trie/src/lib.rs | 4 +- test-support/reference-trie/src/lib.rs | 31 +++++++---- trie-db/src/lib.rs | 10 +++- trie-db/src/node_codec.rs | 74 +++++++++++++++++++++----- trie-db/src/proof/verify.rs | 3 +- trie-db/src/trie_codec.rs | 2 +- 6 files changed, 95 insertions(+), 29 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index fb27ae2e..3e2289ec 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -1225,7 +1225,7 @@ mod test { fn test_depth_iter() { // cases corresponding to test_depth, TODO add more // let cases = [7, 6, 5, 12, 11, 10, 9]; - for nb in (0usize..16) { + for nb in 0usize..16 { let mut n = 0; let tree = Tree::new(0, 0, nb); for (i, (d, k)) in tree.iter_depth(None) @@ -1410,7 +1410,6 @@ mod test { &mut callback_read_proof, false, ); - let additional_hash = callback.additional_hash; println!("{}, {}", l, p); assert!(root.is_some()); assert_eq!(root.unwrap().as_ref(), &result[l][..]); @@ -1463,7 +1462,6 @@ mod test { &mut callback_read_proof, false, ); - let additional_hash = callback.additional_hash; println!("{}, {:?}", l, ps); assert!(root.is_some()); assert_eq!(root.unwrap().as_ref(), &result[l][..]); diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 2059ca90..37e490d0 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -31,13 +31,14 @@ use trie_db::{ Partial, BinaryHasher, EncodedNoChild, + HashesPlan, }; use std::borrow::Borrow; use keccak_hasher::KeccakHasher; pub use trie_db::{ decode_compact, encode_compact, HashDBComplexDyn, - nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, + nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, NodeCodecComplex, Trie, TrieConfiguration, TrieDB, TrieDBIterator, TrieDBMut, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, TrieMut, Bitmap, BITMAP_LENGTH, }; @@ -727,10 +728,6 @@ impl NodeCodec for ReferenceNodeCodec { Ok(Self::decode_plan_internal(data, false)?.0) } - fn decode_plan_no_child(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { - Self::decode_plan_internal(data, true) - } - fn is_empty_node(data: &[u8]) -> bool { data == ::empty_node() } @@ -840,6 +837,16 @@ impl NodeCodec for ReferenceNodeCodec { } +impl NodeCodecComplex for ReferenceNodeCodec { + type AdditionalHashesPlan = HashesPlan; + fn decode_plan_proof(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { + Self::decode_plan_internal(data, true) + } + fn decode_proof(_data: &[u8]) -> ::std::result::Result<(NodePlan, HashesPlan), Self::Error> { + unimplemented!() + } +} + impl ReferenceNodeCodecNoExt { fn decode_plan_internal( data: &[u8], @@ -921,10 +928,6 @@ impl NodeCodec for ReferenceNodeCodecNoExt { Ok(Self::decode_plan_internal(data, false)?.0) } - fn decode_plan_no_child(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { - Self::decode_plan_internal(data, true) - } - fn hashed_null_node() -> ::Out { H::hash(::empty_node()) } @@ -1038,6 +1041,16 @@ impl NodeCodec for ReferenceNodeCodecNoExt { } +impl NodeCodecComplex for ReferenceNodeCodecNoExt { + type AdditionalHashesPlan = HashesPlan; + fn decode_plan_proof(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { + Self::decode_plan_internal(data, true) + } + fn decode_proof(_data: &[u8]) -> ::std::result::Result<(NodePlan, HashesPlan), Self::Error> { + unimplemented!() + } +} + /// Compare trie builder and in memory trie. pub fn compare_implementations + ordered_trie::HashDBComplex + Eq> ( data: Vec<(Vec, Vec)>, diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index e47214d5..dcada03c 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -70,7 +70,8 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, EncodedNoChild, Bitmap, BITMAP_LENGTH}; +pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, EncodedNoChild, + Bitmap, BITMAP_LENGTH, HashesPlan}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; pub use crate::iterator::TrieDBNodeIterator; @@ -383,11 +384,16 @@ pub trait TrieLayout { /// no partial in branch, if false the trie will only /// use branch and node with partials in both. const USE_EXTENSION: bool; + /// Does the layout implement a complex hash. + /// Note that if does not, the `NodeCodecComplex` hash + /// associated codec only really need to implement `NodeCodec` + /// and dummy implementation can be used. + /// TODO EMCH consider splitting trait with a TrieLayoutComplex variant!! const COMPLEX_HASH: bool; /// Hasher to use for this trie. type Hash: BinaryHasher; /// Codec to use (needs to match hasher and nibble ops). - type Codec: NodeCodec::Out>; + type Codec: NodeCodecComplex::Out>; } /// This trait associates a trie definition with preferred methods. diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index b67f85a6..8b8896db 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -19,7 +19,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData, mem::replace}; +use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData, mem::replace, iter::from_fn}; /// Representation of a nible slice (right aligned). /// It contains a right aligned padded first byte (first pair element is the number of nibbles @@ -42,9 +42,6 @@ pub trait NodeCodec: Sized { /// Decode bytes to a `NodePlan`. Returns `Self::E` on failure. fn decode_plan(data: &[u8]) -> Result; - /// See `Decode_no_child`. - fn decode_plan_no_child(data: &[u8]) -> Result<(NodePlan, usize), Self::Error>; - /// Decode bytes to a `Node`. Returns `Self::E` on failure. fn decode(data: &[u8]) -> Result { // TODO ensure real use codec have their own implementation @@ -52,13 +49,6 @@ pub trait NodeCodec: Sized { Ok(Self::decode_plan(data)?.build(data)) } - /// Decode but child are not include (instead we put empty inline - /// nodes). - fn decode_no_child(data: &[u8]) -> Result<(Node, usize), Self::Error> { - let (plan, offset) = Self::decode_plan_no_child(data)?; - Ok((plan.build(data), offset)) - } - /// Check if the provided bytes correspond to the codecs "empty" node. fn is_empty_node(data: &[u8]) -> bool; @@ -108,7 +98,7 @@ pub type OffsetChildren = usize; /// - Intermediate optional common representation shared between storage pub trait NodeCodecComplex: NodeCodec { /// Sequence of hashes needed for the children proof verification. - type AdditionalHashes: Iterator; + type AdditionalHashesPlan: Iterator>; /// `Decode_no_child`, returning an offset position if there is a common representation. /// NodePlan do not include child (sized null). @@ -124,10 +114,18 @@ pub trait NodeCodecComplex: NodeCodec { Ok((plan.build(data), offset)) } - fn decode_proof(data: &[u8]) -> Result<(NodePlan, Self::AdditionalHashes), Self::Error>; + fn decode_proof(data: &[u8]) -> Result<(NodePlan, Self::AdditionalHashesPlan), Self::Error>; + /// Decode but child are not include (instead we put empty inline + /// nodes). + fn decode_no_child(data: &[u8]) -> Result<(Node, usize), Self::Error> { + let (plan, offset) = Self::decode_plan_proof(data)?; + Ok((plan.build(data), offset)) + } } + + #[derive(Clone)] pub enum EncodedNoChild { // not runing complex @@ -322,3 +320,53 @@ impl Bitmap { output[1] = (bitmap / 256) as u8; } } + + +/// Simple implementation of a additional hash iterator based +/// upon a sequential encoding of known length. +pub struct HashesPlan { + hash_len: usize, + /// we use two size counter to implement `size_hint`. + end: usize, + offset: usize, +} + +impl HashesPlan { + pub fn new(nb_child: usize, offset: usize, hash_len: usize) -> Self { + HashesPlan { + end: offset + (hash_len * nb_child), + hash_len, + offset, + } + } + pub fn iter_hashes<'a, HO: Default + AsMut<[u8]>>(mut self, data: &'a [u8]) -> impl Iterator + 'a { + from_fn(move || { + self.next().map(|range| { + let mut result = HO::default(); + result.as_mut().copy_from_slice(&data[range]); + result + }) + }) + } +} + +impl Iterator for HashesPlan { + type Item = Range; + + fn next(&mut self) -> Option { + if self.offset < self.end { + self.end += self.hash_len; + Some(Range { + start: self.end - self.hash_len, + end: self.end + }) + } else { + None + } + } + + fn size_hint(&self) -> (usize, Option) { + let size = self.end / self.hash_len; + (size, Some(size)) + } +} diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index deac9439..713621d3 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -19,6 +19,7 @@ use crate::rstd::{ use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle}, NodeCodec, TrieHash, TrieLayout, EncodedNoChild, + NodeCodecComplex, }; use hash_db::Hasher; use ordered_trie::{BinaryHasher, HasherComplex}; @@ -113,7 +114,7 @@ struct StackEntry<'a, C: NodeCodec, H> { _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> +impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> where H: BinaryHasher, { diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index fc7b8d78..97828814 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -28,7 +28,7 @@ pub use ordered_trie::{BinaryHasher, HashDBComplex}; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, - TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, + TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecComplex, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; use crate::node_codec::{Bitmap, BITMAP_LENGTH, EncodedNoChild}; From 63b899b0a1eba9437cc5ffde7a6ee40c0a27d06d Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Apr 2020 16:44:53 +0200 Subject: [PATCH 04/73] Decode for proof. --- test-support/reference-trie/src/lib.rs | 66 +++++++++++++++++---- trie-db/src/node_codec.rs | 82 ++++++++++++++++---------- trie-db/src/proof/verify.rs | 68 +++------------------ trie-db/src/trie_codec.rs | 63 ++------------------ 4 files changed, 117 insertions(+), 162 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 37e490d0..665703ed 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -839,14 +839,61 @@ impl NodeCodec for ReferenceNodeCodec { impl NodeCodecComplex for ReferenceNodeCodec { type AdditionalHashesPlan = HashesPlan; - fn decode_plan_proof(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { - Self::decode_plan_internal(data, true) - } - fn decode_proof(_data: &[u8]) -> ::std::result::Result<(NodePlan, HashesPlan), Self::Error> { - unimplemented!() + + fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { + let (node, offset) = Self::decode_plan_internal(data, true)?; + decode_plan_proof_internal(data, offset, node, H::LENGTH) } } +fn decode_plan_proof_internal( + data: &[u8], + mut offset: usize, + mut node: NodePlan, + hash_len: usize, +) -> Result<(NodePlan, Option<(Bitmap, HashesPlan)>), CodecError> { + let hashes_plan = match &mut node { + NodePlan::Branch{children, ..} | NodePlan::NibbledBranch{children, ..} => { + if data.len() < offset + 3 { + return Err(CodecError::from("Decode branch, missing proof headers")); + } + // TODO EMCH bitmap looks unused!!! + let keys_position = Bitmap::decode(&data[offset..offset + BITMAP_LENGTH]); + offset += BITMAP_LENGTH; + + let nb_additional; + // read inline nodes. + loop { + let nb = data[offset] as usize; + offset += 1; + if nb >= 128 { + nb_additional = nb - 128; + break; + } + // 2 for inline index and next elt length. + if data.len() < offset + nb + 2 { + return Err(CodecError::from("Decode branch, missing proof inline data")); + } + let ix = data[offset] as usize; + offset += 1; + let inline = offset..offset + nb; + if ix >= nibble_ops::NIBBLE_LENGTH { + return Err(CodecError::from("Decode branch, invalid inline index")); + } + children[ix] = Some(NodeHandlePlan::Inline(inline)); + offset += nb; + } + let additional_len = nb_additional * hash_len; + if data.len() < offset + additional_len { + return Err(CodecError::from("Decode branch, missing child proof hashes")); + } + Some((keys_position, HashesPlan::new(nb_additional, offset, hash_len))) + }, + _ => None, + }; + Ok((node, hashes_plan)) +} + impl ReferenceNodeCodecNoExt { fn decode_plan_internal( data: &[u8], @@ -1043,11 +1090,10 @@ impl NodeCodec for ReferenceNodeCodecNoExt { impl NodeCodecComplex for ReferenceNodeCodecNoExt { type AdditionalHashesPlan = HashesPlan; - fn decode_plan_proof(data: &[u8]) -> ::std::result::Result<(NodePlan, usize), Self::Error> { - Self::decode_plan_internal(data, true) - } - fn decode_proof(_data: &[u8]) -> ::std::result::Result<(NodePlan, HashesPlan), Self::Error> { - unimplemented!() + + fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { + let (node, offset) = Self::decode_plan_internal(data, true)?; + decode_plan_proof_internal(data, offset, node, H::LENGTH) } } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 8b8896db..007db040 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -19,7 +19,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData, mem::replace, iter::from_fn}; +use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData, mem::replace}; /// Representation of a nible slice (right aligned). /// It contains a right aligned padded first byte (first pair element is the number of nibbles @@ -87,9 +87,6 @@ pub trait NodeCodec: Sized { ) -> (Vec, EncodedNoChild); } -/// Positional input for children decoding. -pub type OffsetChildren = usize; - /// Trait for handling complex proof. /// This adds methods to basic node codec in order to support: /// - storage encoding with existing `NodeCodec` methods @@ -103,29 +100,24 @@ pub trait NodeCodecComplex: NodeCodec { /// `Decode_no_child`, returning an offset position if there is a common representation. /// NodePlan do not include child (sized null). /// TODO EMCH this is technical function for common implementation. - fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, OffsetChildren), Self::Error>; - - /// Decode but child are not included (instead we put empty inline - /// nodes). - /// An children positianal information is also return for decoding of children. - /// TODO EMCH this looks rather useless. - fn decode_node_proof(data: &[u8]) -> Result<(Node, OffsetChildren), Self::Error> { - let (plan, offset) = Self::decode_plan_proof(data)?; - Ok((plan.build(data), offset)) - } - - fn decode_proof(data: &[u8]) -> Result<(NodePlan, Self::AdditionalHashesPlan), Self::Error>; + /// TODO document this damn bitmap!!! + fn decode_plan_proof(data: &[u8]) -> Result<( + NodePlan, + Option<(Bitmap, Self::AdditionalHashesPlan)>, + ), Self::Error>; /// Decode but child are not include (instead we put empty inline /// nodes). - fn decode_no_child(data: &[u8]) -> Result<(Node, usize), Self::Error> { - let (plan, offset) = Self::decode_plan_proof(data)?; - Ok((plan.build(data), offset)) + fn decode_proof(data: &[u8]) -> Result<( + Node, + Option<(Bitmap, HashesIter)>, + ), Self::Error> { + let (plan, hashes) = Self::decode_plan_proof(data)?; + let hashes = hashes.map(|(bitmap, hashes)| (bitmap, HashesIter::new(data, hashes))); + Ok((plan.build(data), hashes)) } } - - #[derive(Clone)] pub enum EncodedNoChild { // not runing complex @@ -339,14 +331,40 @@ impl HashesPlan { offset, } } - pub fn iter_hashes<'a, HO: Default + AsMut<[u8]>>(mut self, data: &'a [u8]) -> impl Iterator + 'a { - from_fn(move || { - self.next().map(|range| { - let mut result = HO::default(); - result.as_mut().copy_from_slice(&data[range]); - result - }) - }) +} + +/// Iterator over additional hashes +/// upon a sequential encoding of known length. +pub struct HashesIter<'a, I, HO> { + data: &'a [u8], + ranges: I, + buffer: HO, +} + +impl<'a, I, HO: Default> HashesIter<'a, I, HO> { + pub fn new(data: &'a [u8], ranges: I) -> Self { + HashesIter { + ranges, + data, + buffer: HO::default(), + } + } +} + +impl<'a, I, HO> Iterator for HashesIter<'a, I, HO> + where + I: Iterator>, + HO: AsMut<[u8]> + Clone, +{ + type Item = HO; + + fn next(&mut self) -> Option { + if let Some(range) = self.ranges.next() { + self.buffer.as_mut().copy_from_slice(&self.data[range]); + Some(self.buffer.clone()) + } else { + None + } } } @@ -355,10 +373,10 @@ impl Iterator for HashesPlan { fn next(&mut self) -> Option { if self.offset < self.end { - self.end += self.hash_len; + self.offset += self.hash_len; Some(Range { - start: self.end - self.hash_len, - end: self.end + start: self.offset - self.hash_len, + end: self.offset }) } else { None diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index 713621d3..e7570888 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -14,16 +14,14 @@ use crate::rstd::{ convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec, vec::Vec, - iter::from_fn, }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{Node, NodeHandle}, NodeCodec, TrieHash, TrieLayout, EncodedNoChild, - NodeCodecComplex, + node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, EncodedNoChild, }; use hash_db::Hasher; use ordered_trie::{BinaryHasher, HasherComplex}; -use crate::node_codec::{Bitmap, BITMAP_LENGTH}; +use crate::node_codec::{Bitmap, HashesIter}; /// Errors that may occur during proof verification. Most of the errors types simply indicate that @@ -98,7 +96,7 @@ impl std::error::Error for } } -struct StackEntry<'a, C: NodeCodec, H> { +struct StackEntry<'a, C: NodeCodecComplex, H> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, node: Node<'a>, @@ -110,7 +108,8 @@ struct StackEntry<'a, C: NodeCodec, H> { child_index: usize, /// The child references to use in reconstructing the trie nodes. children: Vec>>, - complex: Option<(Bitmap, Vec)>, + /// Proof info if a complex proof is needed. + complex: Option<(Bitmap, HashesIter<'a, C::AdditionalHashesPlan, C::HashOut>)>, _marker: PhantomData<(C, H)>, } @@ -125,60 +124,8 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> let (node, complex) = if !is_inline && complex { // TODO factorize with trie_codec let encoded_node = node_data; - let (mut node, mut offset) = C::decode_no_child(encoded_node) - .map_err(Error::DecodeError)?; - match &mut node { - Node::Branch(b_child, _) | Node::NibbledBranch(_, b_child, _) => { - if encoded_node.len() < offset + 3 { - // TODO new error or move this parte to codec trait and use codec error - return Err(Error::IncompleteProof); - } - let keys_position = Bitmap::decode(&encoded_node[offset..offset + BITMAP_LENGTH]); - offset += BITMAP_LENGTH; - - let mut nb_additional; - // inline nodes - loop { - let nb = encoded_node[offset] as usize; - offset += 1; - if nb >= 128 { - nb_additional = nb - 128; - break; - } - if encoded_node.len() < offset + nb + 2 { - return Err(Error::IncompleteProof); - } - let ix = encoded_node[offset] as usize; - offset += 1; - let inline = &encoded_node[offset..offset + nb]; - if ix >= NIBBLE_LENGTH { - return Err(Error::IncompleteProof); - } - b_child[ix] = Some(NodeHandle::Inline(inline)); - offset += nb; - } - let hash_len = ::NULL_HASH.len(); - let additional_len = nb_additional * hash_len; - if encoded_node.len() < offset + additional_len { - return Err(Error::IncompleteProof); - } - let additional_hashes = from_fn(move || { - if nb_additional > 0 { - let mut hash = ::default(); - hash.as_mut().copy_from_slice(&encoded_node[offset..offset + hash_len]); - offset += hash_len; - nb_additional -= 1; - Some(hash) - } else { - None - } - }); - // TODO dedicated iterator type instead of from_fn to avoid alloc - let additional_hashes: Vec = additional_hashes.collect(); - (node, Some((keys_position, additional_hashes))) - }, - _ => (node, None), - } + C::decode_proof(encoded_node) + .map_err(Error::DecodeError)? } else { (C::decode(node_data) .map_err(Error::DecodeError)?, None) @@ -189,7 +136,6 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> Node::Branch(_, value) | Node::NibbledBranch(_, _, value) => value, }; - Ok(StackEntry { node, is_inline, diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 97828814..8d487ff8 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -31,10 +31,10 @@ use crate::{ TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecComplex, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; -use crate::node_codec::{Bitmap, BITMAP_LENGTH, EncodedNoChild}; +use crate::node_codec::{Bitmap, EncodedNoChild}; use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, rc::Rc, result, vec, vec::Vec, - iter::from_fn, ops::Range, + ops::Range, }; use ordered_trie::{SequenceBinaryTree, HashProof, trie_root, UsizeKeyNode}; struct EncoderStackEntry { @@ -516,63 +516,8 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) for (i, encoded_node) in encoded.iter().enumerate() { let (node, complex) = if L::COMPLEX_HASH { - let (mut node, mut offset) = L::Codec::decode_no_child(encoded_node) - .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))?; - match &mut node { - Node::Branch(b_child, _) | Node::NibbledBranch(_, b_child, _) => { - if encoded_node.len() < offset + 3 { - // TODO new error or move this parte to codec trait and use codec error - return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); - } - let keys_position = Bitmap::decode(&encoded_node[offset..offset + BITMAP_LENGTH]); - offset += BITMAP_LENGTH; - - let mut nb_additional; - // inline nodes - loop { - let nb = encoded_node[offset] as usize; - offset += 1; - if nb >= 128 { - nb_additional = nb - 128; - break; - } - if encoded_node.len() < offset + nb + 2 { - // TODO new error or move this parte to codec trait and use codec error - return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); - } - let ix = encoded_node[offset] as usize; - offset += 1; - let inline = &encoded_node[offset..offset + nb]; - if ix >= NIBBLE_LENGTH { - // TODO new error or move this parte to codec trait and use codec error - return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); - } - b_child[ix] = Some(NodeHandle::Inline(inline)); - offset += nb; - } - let hash_len = ::NULL_HASH.len(); - let additional_len = nb_additional * hash_len; - if encoded_node.len() < offset + additional_len { - // TODO new error or move this parte to codec trait and use codec error - return Err(Box::new(TrieError::IncompleteDatabase(>::default()))); - } - let additional_hashes = from_fn(move || { - if nb_additional > 0 { - let mut hash = >::default(); - hash.as_mut().copy_from_slice(&encoded_node[offset..offset + hash_len]); - offset += hash_len; - nb_additional -= 1; - Some(hash) - } else { - None - } - }); - (node, Some((keys_position, additional_hashes))) - }, - _ => { - (node, None) - }, - } + L::Codec::decode_proof(encoded_node) + .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))? } else { ( L::Codec::decode(encoded_node) From 5d6815715e4c0916876916ee10a27be8c26ccae8 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Apr 2020 17:57:42 +0200 Subject: [PATCH 05/73] split encoding functions. --- test-support/reference-trie/src/lib.rs | 99 ++++++++++++++++++++------ trie-db/src/iter_build.rs | 39 ++++++---- trie-db/src/node_codec.rs | 26 +++++-- trie-db/src/proof/generate.rs | 20 +++--- trie-db/src/proof/verify.rs | 8 +-- trie-db/src/trie_codec.rs | 70 ++++++++++++------ trie-db/src/triedbmut.rs | 80 +++++++++++++-------- 7 files changed, 237 insertions(+), 105 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 665703ed..78ab34cb 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -764,6 +764,52 @@ impl NodeCodec for ReferenceNodeCodec { fn branch_node( children: impl Iterator>>>, maybe_value: Option<&[u8]>, + ) -> Vec { + Self::branch_node_internal(children, maybe_value, None).0 + } + + fn branch_node_nibbled( + _partial: impl Iterator, + _number_nibble: usize, + _children: impl Iterator>>>, + _maybe_value: Option<&[u8]>, + ) -> Vec { + unreachable!() + } + +} + +impl NodeCodecComplex for ReferenceNodeCodec { + type AdditionalHashesPlan = HashesPlan; + + fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { + let (node, offset) = Self::decode_plan_internal(data, true)?; + decode_plan_proof_internal(data, offset, node, H::LENGTH) + } + + fn branch_node_proof( + children: impl Iterator>>>, + maybe_value: Option<&[u8]>, + register_children: &mut [Option>], + ) -> (Vec, EncodedNoChild) { + Self::branch_node_internal(children, maybe_value, Some(register_children)) + } + + fn branch_node_nibbled_proof( + _partial: impl Iterator, + _number_nibble: usize, + _children: impl Iterator>>>, + _maybe_value: Option<&[u8]>, + _register_children: &mut [Option>], + ) -> (Vec, EncodedNoChild) { + unreachable!() + } +} + +impl ReferenceNodeCodec { + fn branch_node_internal( + children: impl Iterator::HashOut>>>>, + maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, ) -> (Vec, EncodedNoChild) { let mut output = vec![0; BITMAP_LENGTH + 1]; @@ -824,26 +870,6 @@ impl NodeCodec for ReferenceNodeCodec { output[0..BITMAP_LENGTH + 1].copy_from_slice(prefix.as_ref()); (output, no_child) } - - fn branch_node_nibbled( - _partial: impl Iterator, - _number_nibble: usize, - _children: impl Iterator>>>, - _maybe_value: Option<&[u8]>, - _register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild) { - unreachable!() - } - -} - -impl NodeCodecComplex for ReferenceNodeCodec { - type AdditionalHashesPlan = HashesPlan; - - fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { - let (node, offset) = Self::decode_plan_internal(data, true)?; - decode_plan_proof_internal(data, offset, node, H::LENGTH) - } } fn decode_plan_proof_internal( @@ -1004,8 +1030,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { fn branch_node( _children: impl Iterator::Out>>>>, _maybe_value: Option<&[u8]>, - _register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild) { + ) -> Vec { unreachable!() } @@ -1014,6 +1039,17 @@ impl NodeCodec for ReferenceNodeCodecNoExt { number_nibble: usize, children: impl Iterator>>>, maybe_value: Option<&[u8]>, + ) -> Vec { + Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, None).0 + } +} + +impl ReferenceNodeCodecNoExt { + fn branch_node_nibbled_internal( + partial: impl Iterator, + number_nibble: usize, + children: impl Iterator::HashOut>>>>, + maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, ) -> (Vec, EncodedNoChild) { let mut output = if maybe_value.is_some() { @@ -1085,7 +1121,6 @@ impl NodeCodec for ReferenceNodeCodecNoExt { .copy_from_slice(&bitmap.as_ref()[..BITMAP_LENGTH]); (output, no_child) } - } impl NodeCodecComplex for ReferenceNodeCodecNoExt { @@ -1095,6 +1130,24 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { let (node, offset) = Self::decode_plan_internal(data, true)?; decode_plan_proof_internal(data, offset, node, H::LENGTH) } + + fn branch_node_proof( + _children: impl Iterator::Out>>>>, + _maybe_value: Option<&[u8]>, + _register_children: &mut [Option>], + ) -> (Vec, EncodedNoChild) { + unreachable!() + } + + fn branch_node_nibbled_proof( + partial: impl Iterator, + number_nibble: usize, + children: impl Iterator>>>, + maybe_value: Option<&[u8]>, + register_children: &mut [Option>], + ) -> (Vec, EncodedNoChild) { + Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, Some(register_children)) + } } /// Compare trie builder and in memory trie. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 9908a6c4..7a003d14 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -23,7 +23,7 @@ use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; use crate::nibble::nibble_ops; -use crate::node_codec::{NodeCodec, EncodedNoChild}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedNoChild}; use crate::{TrieLayout, TrieHash}; use crate::rstd::borrow::Borrow; @@ -209,11 +209,18 @@ impl CacheAccum let v = self.0[last].1.take(); let mut register_children = register_children_buf::(); - let encoded = T::Codec::branch_node( - self.0[last].0.as_ref().iter(), - v.as_ref().map(|v| v.as_ref()), - register_children.as_mut().map(|t| t.as_mut()), // TODO switch fully to codec approach or just return trim info - ); + let encoded = if let Some(register_children) = register_children.as_mut() { + T::Codec::branch_node_proof( + self.0[last].0.as_ref().iter(), + v.as_ref().map(|v| v.as_ref()), + register_children.as_mut() + ) + } else { + (T::Codec::branch_node( + self.0[last].0.as_ref().iter(), + v.as_ref().map(|v| v.as_ref()), + ), EncodedNoChild::Unused) + }; let pr = NibbleSlice::new_offset(&key_branch, branch_d); let branch_hash = if T::COMPLEX_HASH { let len = self.0[last].0.as_ref().iter().filter(|v| v.is_some()).count(); @@ -253,12 +260,20 @@ impl CacheAccum let nkeyix = nkey.unwrap_or((0, 0)); let mut register_children = register_children_buf::(); let pr = NibbleSlice::new_offset(&key_branch, nkeyix.0); - let encoded = T::Codec::branch_node_nibbled( - pr.right_range_iter(nkeyix.1), - nkeyix.1, - self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - register_children.as_mut().map(|t| t.as_mut()), // TODO switch fully to codec approach - ); + let encoded = if let Some(register_children) = register_children.as_mut() { + T::Codec::branch_node_nibbled_proof( + pr.right_range_iter(nkeyix.1), + nkeyix.1, + self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), + register_children.as_mut(), + ) + } else { + (T::Codec::branch_node_nibbled( + pr.right_range_iter(nkeyix.1), + nkeyix.1, + self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), + ), EncodedNoChild::Unused) + }; let ext_length = nkey.as_ref().map(|nkeyix| nkeyix.0).unwrap_or(0); let pr = NibbleSlice::new_offset( &key_branch, diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 007db040..594d4115 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -73,8 +73,7 @@ pub trait NodeCodec: Sized { fn branch_node( children: impl Iterator>>>, value: Option<&[u8]>, - register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild); + ) -> Vec; /// Returns an encoded branch node with a possible partial path. /// `number_nibble` is the partial path length as in `extension_node`. @@ -82,9 +81,8 @@ pub trait NodeCodec: Sized { partial: impl Iterator, number_nibble: usize, children: impl Iterator>>>, - value: Option<&[u8]>, - register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild); + value: Option<&[u8]> + ) -> Vec; } /// Trait for handling complex proof. @@ -116,6 +114,24 @@ pub trait NodeCodecComplex: NodeCodec { let hashes = hashes.map(|(bitmap, hashes)| (bitmap, HashesIter::new(data, hashes))); Ok((plan.build(data), hashes)) } + + /// Returns an encoded branch node. + /// Takes an iterator yielding `ChildReference` and an optional value. + fn branch_node_proof( + children: impl Iterator>>>, + value: Option<&[u8]>, + register_children: &mut [Option>], + ) -> (Vec, EncodedNoChild); + + /// Returns an encoded branch node with a possible partial path. + /// `number_nibble` is the partial path length as in `extension_node`. + fn branch_node_nibbled_proof( + partial: impl Iterator, + number_nibble: usize, + children: impl Iterator>>>, + value: Option<&[u8]>, + register_children: &mut [Option>], + ) -> (Vec, EncodedNoChild); } #[derive(Clone)] diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index d5b16fbf..04b578af 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -24,7 +24,7 @@ use crate::node_codec::Bitmap; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, Result as TrieResult, Trie, TrieError, TrieHash, - TrieLayout, + TrieLayout, NodeCodecComplex, }; use ordered_trie::BinaryHasher; @@ -47,7 +47,7 @@ struct StackEntry<'a, C: NodeCodec, H> { _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> +impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> where H: BinaryHasher, { @@ -120,10 +120,10 @@ impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> if !self.is_inline && complex { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - let (mut result, no_child) = C::branch_node( + let (mut result, no_child) = C::branch_node_proof( self.children.iter(), value_with_omission(node_data, value, self.omit_value), - Some(register_children), + register_children, ); no_child.trim_no_child(&mut result); let bitmap_start = result.len(); @@ -158,8 +158,7 @@ impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> C::branch_node( self.children.into_iter(), value_with_omission(node_data, value, self.omit_value), - None, // TODO allow complex here - ).0 + ) } }, NodePlan::NibbledBranch { partial: partial_plan, value, children } => { @@ -174,12 +173,12 @@ impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> // TODO factor with non nibbled!! let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - let (mut result, no_child) = C::branch_node_nibbled( + let (mut result, no_child) = C::branch_node_nibbled_proof( partial.right_iter(), partial.len(), self.children.iter(), value_with_omission(node_data, value, self.omit_value), - Some(register_children), + register_children, ); no_child.trim_no_child(&mut result); let bitmap_start = result.len(); @@ -216,8 +215,7 @@ impl<'a, C: NodeCodec, H: BinaryHasher> StackEntry<'a, C, H> partial.len(), self.children.into_iter(), value_with_omission(node_data, value, self.omit_value), - None, // TODO allow complex here - ).0 + ) } }, }) @@ -596,7 +594,7 @@ fn value_with_omission<'a>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( +fn unwind_stack( stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index e7570888..c9080651 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -175,21 +175,21 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> Node::Branch(_, _) => { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - C::branch_node( + C::branch_node_proof( self.children.iter(), self.value, - Some(register_children), // TODO again unused register result + register_children, // TODO unused register result ) }, Node::NibbledBranch(partial, _, _) => { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - C::branch_node_nibbled( + C::branch_node_nibbled_proof( partial.right_iter(), partial.len(), self.children.iter(), self.value, - Some(register_children), // TODO again unused register result + register_children, // TODO again unused register result ) }, }) diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 8d487ff8..1b3beb74 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -52,7 +52,7 @@ struct EncoderStackEntry { _marker: PhantomData, } -impl EncoderStackEntry { +impl EncoderStackEntry { /// Given the prefix of the next child node, identify its index and advance `child_index` to /// that. For a given entry, this must be called sequentially only with strictly increasing /// child prefixes. Returns an error if the child prefix is not a child of this entry or if @@ -126,11 +126,18 @@ impl EncoderStackEntry { } else { (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) }; - let (mut result, no_child) = C::branch_node( - children.iter(), - value.clone().map(|range| &node_data[range]), - complex, - ); + let (mut result, no_child) = if let Some(complex) = complex { + C::branch_node_proof( + children.iter(), + value.clone().map(|range| &node_data[range]), + complex, + ) + } else { + (C::branch_node( + children.iter(), + value.clone().map(|range| &node_data[range]), + ), EncodedNoChild::Unused) + }; if complex_hash { no_child.trim_no_child(&mut result); let bitmap_start = result.len(); @@ -171,13 +178,22 @@ impl EncoderStackEntry { (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) }; let partial = partial.build(node_data); - let (mut result, no_child) = C::branch_node_nibbled( - partial.right_iter(), - partial.len(), - children.iter(), - value.clone().map(|range| &node_data[range]), - complex, - ); + let (mut result, no_child) = if let Some(complex) = complex { + C::branch_node_nibbled_proof( + partial.right_iter(), + partial.len(), + children.iter(), + value.clone().map(|range| &node_data[range]), + complex, + ) + } else { + (C::branch_node_nibbled( + partial.right_iter(), + partial.len(), + children.iter(), + value.clone().map(|range| &node_data[range]), + ), EncodedNoChild::Unused) + }; if complex_hash { no_child.trim_no_child(&mut result); let bitmap_start = result.len(); @@ -352,7 +368,7 @@ struct DecoderStackEntry<'a, C: NodeCodec, F> { _marker: PhantomData, } -impl<'a, C: NodeCodec, F> DecoderStackEntry<'a, C, F> { +impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { /// Advance the child index until either it exceeds the number of children or the child is /// marked as omitted. Omitted children are indicated by an empty inline reference. For each /// child that is passed over and not omitted, copy over the child reference from the node to @@ -471,20 +487,34 @@ impl<'a, C: NodeCodec, F> DecoderStackEntry<'a, C, F> { self.children[0] .expect("required by method precondition; qed"), ), EncodedNoChild::Unused), - Node::Branch(_, value) => - C::branch_node( + Node::Branch(_, value) => if let Some(register_children) = register_children { + C::branch_node_proof( self.children.into_iter(), value, register_children, - ), - Node::NibbledBranch(partial, _, value) => - C::branch_node_nibbled( + ) + } else { + (C::branch_node( + self.children.into_iter(), + value, + ), EncodedNoChild::Unused) + }, + Node::NibbledBranch(partial, _, value) => if let Some(register_children) = register_children { + C::branch_node_nibbled_proof( partial.right_iter(), partial.len(), self.children.iter(), value, register_children, - ), + ) + } else { + (C::branch_node_nibbled( + partial.right_iter(), + partial.len(), + self.children.iter(), + value, + ), EncodedNoChild::Unused) + }, } } } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index a273763e..852b993e 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -23,7 +23,7 @@ use crate::node_codec::HashDBComplexDyn; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::node_codec::{NodeCodec, EncodedNoChild}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedNoChild}; use crate::nibble::{NibbleVec, NibbleSlice, nibble_ops, BackingByteVec}; use crate::rstd::{ boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, @@ -210,7 +210,7 @@ where mut child_cb: F, register_children: Option<&mut [Option>]>, ) -> (Vec, EncodedNoChild) where - C: NodeCodec, + C: NodeCodecComplex, F: FnMut(NodeHandle, Option<&NibbleSlice>, Option) -> ChildReference, H: Hasher, { @@ -231,38 +231,58 @@ where ), EncodedNoChild::Unused) }, Node::Branch(mut children, value) => { - C::branch_node( - // map the `NodeHandle`s from the Branch to `ChildReferences` - children.iter_mut() - .map(Option::take) - .enumerate() - .map(|(i, maybe_child)| { - maybe_child.map(|child| child_cb(child, None, Some(i as u8))) - }), - value.as_ref().map(|v| &v[..]), - register_children, - ) + // map the `NodeHandle`s from the Branch to `ChildReferences` + let children = children.iter_mut() + .map(Option::take) + .enumerate() + .map(|(i, maybe_child)| { + maybe_child.map(|child| child_cb(child, None, Some(i as u8))) + }); + if let Some(register_children) = register_children { + C::branch_node_proof( + // map the `NodeHandle`s from the Branch to `ChildReferences` + children, + value.as_ref().map(|v| &v[..]), + register_children, + ) + } else { + (C::branch_node( + children, + value.as_ref().map(|v| &v[..]), + ), EncodedNoChild::Unused) + } }, Node::NibbledBranch(partial, mut children, value) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let it = pr.right_iter(); - C::branch_node_nibbled( - it, - pr.len(), - // map the `NodeHandle`s from the Branch to `ChildReferences` - children.iter_mut() - .map(Option::take) - .enumerate() - .map(|(i, maybe_child)| { - //let branch_index = [i as u8]; - maybe_child.map(|child| { - let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - child_cb(child, Some(&pr), Some(i as u8)) - }) - }), - value.as_ref().map(|v| &v[..]), - register_children, - ) + // map the `NodeHandle`s from the Branch to `ChildReferences` + let children = children.iter_mut() + .map(Option::take) + .enumerate() + .map(|(i, maybe_child)| { + //let branch_index = [i as u8]; + maybe_child.map(|child| { + let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); + child_cb(child, Some(&pr), Some(i as u8)) + }) + }); + + if let Some(register_children) = register_children { + C::branch_node_nibbled_proof( + it, + pr.len(), + children, + value.as_ref().map(|v| &v[..]), + register_children, + ) + } else { + (C::branch_node_nibbled( + it, + pr.len(), + children, + value.as_ref().map(|v| &v[..]), + ), EncodedNoChild::Unused) + } }, } } From 59d8399794cc1d01b24f3fdc0b4c74c839ff48e3 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Apr 2020 18:42:24 +0200 Subject: [PATCH 06/73] renaming --- test-support/reference-trie/src/lib.rs | 88 ++++++++++++++++---------- trie-db/src/iter_build.rs | 40 ++++++------ trie-db/src/lib.rs | 2 +- trie-db/src/node_codec.rs | 52 +++++++-------- trie-db/src/proof/generate.rs | 8 +-- trie-db/src/proof/verify.rs | 18 +++--- trie-db/src/trie_codec.rs | 38 +++++------ trie-db/src/triedbmut.rs | 18 +++--- 8 files changed, 142 insertions(+), 122 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 78ab34cb..63b1d8f3 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -30,7 +30,7 @@ use trie_db::{ TrieRootComplex, Partial, BinaryHasher, - EncodedNoChild, + EncodedCommon, HashesPlan, }; use std::borrow::Borrow; @@ -765,7 +765,7 @@ impl NodeCodec for ReferenceNodeCodec { children: impl Iterator>>>, maybe_value: Option<&[u8]>, ) -> Vec { - Self::branch_node_internal(children, maybe_value, None).0 + Self::branch_node_internal(children, maybe_value, None, true).0 } fn branch_node_nibbled( @@ -787,21 +787,21 @@ impl NodeCodecComplex for ReferenceNodeCodec { decode_plan_proof_internal(data, offset, node, H::LENGTH) } - fn branch_node_proof( + fn branch_node_common( children: impl Iterator>>>, maybe_value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedNoChild) { - Self::branch_node_internal(children, maybe_value, Some(register_children)) + ) -> (Vec, EncodedCommon) { + Self::branch_node_internal(children, maybe_value, Some(register_children), true) } - fn branch_node_nibbled_proof( + fn branch_node_nibbled_common( _partial: impl Iterator, _number_nibble: usize, _children: impl Iterator>>>, _maybe_value: Option<&[u8]>, _register_children: &mut [Option>], - ) -> (Vec, EncodedNoChild) { + ) -> (Vec, EncodedCommon) { unreachable!() } } @@ -811,7 +811,8 @@ impl ReferenceNodeCodec { children: impl Iterator::HashOut>>>>, maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild) { + encode_children: bool, + ) -> (Vec, EncodedCommon) { let mut output = vec![0; BITMAP_LENGTH + 1]; let mut prefix: [u8; 3] = [0; 3]; let have_value = if let Some(value) = maybe_value { @@ -824,14 +825,16 @@ impl ReferenceNodeCodec { let ix = &mut ix; let mut register_children = register_children.as_mut(); let register_children = &mut register_children; - let no_child = if register_children.is_some() { - EncodedNoChild::Range(Range { + let common = if encode_children && register_children.is_some() { + EncodedCommon::Range(Range { start: 0, end: output.len(), }) } else { - EncodedNoChild::Unused + EncodedCommon::Unused }; + + let mut child_ix = output.len(); let has_children = children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { if let Some(ranges) = register_children { @@ -839,24 +842,30 @@ impl ReferenceNodeCodec { // case for reasonable hash length. let encode_size_offset = 1; ranges[*ix] = Some(Range { - start: output.len() + encode_size_offset, - end: output.len() + encode_size_offset + h.as_ref().len(), + start: child_ix + encode_size_offset, + end: child_ix + encode_size_offset + h.as_ref().len(), }); + child_ix += encode_size_offset + h.as_ref().len(); *ix += 1; } - h.as_ref().encode_to(&mut output); + if encode_children { + h.as_ref().encode_to(&mut output); + } true } &Some(ChildReference::Inline(inline_data, len)) => { if let Some(ranges) = register_children { let encode_size_offset = 1; ranges[*ix] = Some(Range { - start: output.len() + encode_size_offset, - end: output.len() + encode_size_offset + len, + start: child_ix + encode_size_offset, + end: child_ix + encode_size_offset + len, }); + child_ix += encode_size_offset + len; *ix += 1; } - inline_data.as_ref()[..len].encode_to(&mut output); + if encode_children { + inline_data.as_ref()[..len].encode_to(&mut output); + } true } None => { @@ -868,7 +877,7 @@ impl ReferenceNodeCodec { }); branch_node_buffered(have_value, has_children, prefix.as_mut()); output[0..BITMAP_LENGTH + 1].copy_from_slice(prefix.as_ref()); - (output, no_child) + (output, common) } } @@ -1040,7 +1049,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { children: impl Iterator>>>, maybe_value: Option<&[u8]>, ) -> Vec { - Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, None).0 + Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, None, true).0 } } @@ -1051,7 +1060,8 @@ impl ReferenceNodeCodecNoExt { children: impl Iterator::HashOut>>>>, maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild) { + encode_children: bool, + ) -> (Vec, EncodedCommon) { let mut output = if maybe_value.is_some() { partial_from_iterator_encode( partial, @@ -1075,14 +1085,16 @@ impl ReferenceNodeCodecNoExt { let ix = &mut ix; let mut register_children = register_children.as_mut(); let register_children = &mut register_children; - let no_child = if register_children.is_some() { - EncodedNoChild::Range(Range { + let common = if encode_children && register_children.is_some() { + EncodedCommon::Range(Range { start: 0, end: output.len(), }) } else { - EncodedNoChild::Unused + EncodedCommon::Unused }; + + let mut child_ix = output.len(); Bitmap::encode(children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { if let Some(ranges) = register_children { @@ -1090,24 +1102,30 @@ impl ReferenceNodeCodecNoExt { // case for reasonable hash length. let encode_size_offset = 1; ranges[*ix] = Some(Range { - start: output.len() + encode_size_offset, - end: output.len() + encode_size_offset + h.as_ref().len(), + start: child_ix + encode_size_offset, + end: child_ix + encode_size_offset + h.as_ref().len(), }); + child_ix += encode_size_offset + h.as_ref().len(); *ix += 1; } - h.as_ref().encode_to(&mut output); + if encode_children { + h.as_ref().encode_to(&mut output); + } true } &Some(ChildReference::Inline(inline_data, len)) => { if let Some(ranges) = register_children { let encode_size_offset = 1; ranges[*ix] = Some(Range { - start: output.len() + encode_size_offset, - end: output.len() + encode_size_offset + len, + start: child_ix + encode_size_offset, + end: child_ix + encode_size_offset + len, }); + child_ix += encode_size_offset + len; *ix += 1; } - inline_data.as_ref()[..len].encode_to(&mut output); + if encode_children { + inline_data.as_ref()[..len].encode_to(&mut output); + } true } None => { @@ -1119,7 +1137,7 @@ impl ReferenceNodeCodecNoExt { }), bitmap.as_mut()); output[bitmap_index..bitmap_index + BITMAP_LENGTH] .copy_from_slice(&bitmap.as_ref()[..BITMAP_LENGTH]); - (output, no_child) + (output, common) } } @@ -1131,22 +1149,22 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { decode_plan_proof_internal(data, offset, node, H::LENGTH) } - fn branch_node_proof( + fn branch_node_common( _children: impl Iterator::Out>>>>, _maybe_value: Option<&[u8]>, _register_children: &mut [Option>], - ) -> (Vec, EncodedNoChild) { + ) -> (Vec, EncodedCommon) { unreachable!() } - fn branch_node_nibbled_proof( + fn branch_node_nibbled_common( partial: impl Iterator, number_nibble: usize, children: impl Iterator>>>, maybe_value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedNoChild) { - Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, Some(register_children)) + ) -> (Vec, EncodedCommon) { + Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, Some(register_children), true) } } diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 7a003d14..46c4e073 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -23,7 +23,7 @@ use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; use crate::nibble::nibble_ops; -use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedNoChild}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedCommon}; use crate::{TrieLayout, TrieHash}; use crate::rstd::borrow::Borrow; @@ -146,7 +146,7 @@ impl CacheAccum k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); let iter: Option<(EmptyIter>, _)> = None; - let hash = callback.process(pr.left(), (encoded, EncodedNoChild::Unused), false, iter); + let hash = callback.process(pr.left(), (encoded, EncodedCommon::Unused), false, iter); // insert hash in branch (first level branch only at this point) self.set_node(target_depth, nibble_value as usize, Some(hash)); @@ -210,7 +210,7 @@ impl CacheAccum let mut register_children = register_children_buf::(); let encoded = if let Some(register_children) = register_children.as_mut() { - T::Codec::branch_node_proof( + T::Codec::branch_node_common( self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), register_children.as_mut() @@ -219,7 +219,7 @@ impl CacheAccum (T::Codec::branch_node( self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }; let pr = NibbleSlice::new_offset(&key_branch, branch_d); let branch_hash = if T::COMPLEX_HASH { @@ -237,7 +237,7 @@ impl CacheAccum let nib = pr.right_range_iter(nkeyix.1); let encoded = T::Codec::extension_node(nib, nkeyix.1, branch_hash); let iter: Option<(EmptyIter>, _)> = None; - let h = callback.process(pr.left(), (encoded, EncodedNoChild::Unused), is_root, iter); + let h = callback.process(pr.left(), (encoded, EncodedCommon::Unused), is_root, iter); h } else { branch_hash @@ -261,7 +261,7 @@ impl CacheAccum let mut register_children = register_children_buf::(); let pr = NibbleSlice::new_offset(&key_branch, nkeyix.0); let encoded = if let Some(register_children) = register_children.as_mut() { - T::Codec::branch_node_nibbled_proof( + T::Codec::branch_node_nibbled_common( pr.right_range_iter(nkeyix.1), nkeyix.1, self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), @@ -272,7 +272,7 @@ impl CacheAccum pr.right_range_iter(nkeyix.1), nkeyix.1, self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }; let ext_length = nkey.as_ref().map(|nkeyix| nkeyix.0).unwrap_or(0); let pr = NibbleSlice::new_offset( @@ -347,7 +347,7 @@ pub fn trie_visit(input: I, callback: &mut F) k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); let iter: Option<(EmptyIter>, _)> = None; - callback.process(pr.left(), (encoded, EncodedNoChild::Unused), true, iter); + callback.process(pr.left(), (encoded, EncodedCommon::Unused), true, iter); } else { depth_queue.flush_value(callback, last_depth, &previous_value); let ref_branches = previous_value.0; @@ -356,7 +356,7 @@ pub fn trie_visit(input: I, callback: &mut F) } else { let iter: Option<(EmptyIter>, _)> = None; // nothing null root corner case - callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), EncodedNoChild::Unused), true, iter); + callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), EncodedCommon::Unused), true, iter); } } @@ -372,7 +372,7 @@ pub trait ProcessEncodedNode { fn process( &mut self, prefix: Prefix, - encoded_node: (Vec, EncodedNoChild), + encoded_node: (Vec, EncodedCommon), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference; @@ -414,7 +414,7 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> fn process( &mut self, prefix: Prefix, - (encoded_node, _no_child): (Vec, EncodedNoChild), + (encoded_node, _common): (Vec, EncodedCommon), is_root: bool, // TODO different trait?? _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -439,7 +439,7 @@ impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode<, EncodedNoChild), + (encoded_node, common): (Vec, EncodedCommon), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -461,7 +461,7 @@ impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode< ProcessEncodedNode<::Out> for TrieRoot, EncodedNoChild), + (encoded_node, _common): (Vec, EncodedCommon), is_root: bool, // TODO different trait _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -531,7 +531,7 @@ impl ProcessEncodedNode<::Out> for TrieRootComple fn process( &mut self, _: Prefix, - (encoded_node, no_child): (Vec, EncodedNoChild), + (encoded_node, common): (Vec, EncodedCommon), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -550,7 +550,7 @@ impl ProcessEncodedNode<::Out> for TrieRootComple None => None, }); ::hash_complex( - no_child.encoded_no_child(&encoded_node[..]), + common.encoded_common(&encoded_node[..]), nb_children, iter, EmptyIter::default(), @@ -615,7 +615,7 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, EncodedNoChild), + (encoded_node, _common): (Vec, EncodedCommon), is_root: bool, // TODO different trait? _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -643,7 +643,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { fn process( &mut self, _: Prefix, - (encoded_node, _no_child): (Vec, EncodedNoChild), + (encoded_node, _common): (Vec, EncodedCommon), is_root: bool, // TODO different trait _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -667,7 +667,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhash fn process( &mut self, _: Prefix, - (encoded_node, no_child): (Vec, EncodedNoChild), + (encoded_node, common): (Vec, EncodedCommon), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -686,7 +686,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhash None => None, }); ::hash_complex( - no_child.encoded_no_child(&encoded_node[..]), + common.encoded_common(&encoded_node[..]), nb_children, iter, EmptyIter::default(), diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index dcada03c..742890fd 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -70,7 +70,7 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, EncodedNoChild, +pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, EncodedCommon, Bitmap, BITMAP_LENGTH, HashesPlan}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 594d4115..a71b6387 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -19,7 +19,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData, mem::replace}; +use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData}; /// Representation of a nible slice (right aligned). /// It contains a right aligned padded first byte (first pair element is the number of nibbles @@ -95,8 +95,6 @@ pub trait NodeCodecComplex: NodeCodec { /// Sequence of hashes needed for the children proof verification. type AdditionalHashesPlan: Iterator>; - /// `Decode_no_child`, returning an offset position if there is a common representation. - /// NodePlan do not include child (sized null). /// TODO EMCH this is technical function for common implementation. /// TODO document this damn bitmap!!! fn decode_plan_proof(data: &[u8]) -> Result<( @@ -115,57 +113,61 @@ pub trait NodeCodecComplex: NodeCodec { Ok((plan.build(data), hashes)) } - /// Returns an encoded branch node. + /// Returns an encoded branch node, and additional information for getting the common + /// encoded part with the proof base encoding. + /// /// Takes an iterator yielding `ChildReference` and an optional value. - fn branch_node_proof( + fn branch_node_common( children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedNoChild); + ) -> (Vec, EncodedCommon); - /// Returns an encoded branch node with a possible partial path. + /// Returns an encoded branch node with a possible partial path, and additional information + /// for getting the encoding without child common part. /// `number_nibble` is the partial path length as in `extension_node`. - fn branch_node_nibbled_proof( + fn branch_node_nibbled_common( partial: impl Iterator, number_nibble: usize, children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedNoChild); + ) -> (Vec, EncodedCommon); } #[derive(Clone)] -pub enum EncodedNoChild { +pub enum EncodedCommon { // not runing complex Unused, - // range over the full encoded + // range over the encoded common part with the storage encoded Range(Range), // allocated in case we cannot use a range - Allocated(Vec), + NoCommonPart, } -impl EncodedNoChild { - pub fn encoded_no_child<'a>(&'a self, encoded: &'a [u8]) -> &'a [u8] { +impl EncodedCommon { + pub fn encoded_common<'a>(&'a self, encoded: &'a [u8]) -> &'a [u8] { match self { - EncodedNoChild::Unused => encoded, - EncodedNoChild::Range(range) => &encoded[range.clone()], - EncodedNoChild::Allocated(data) => &data[..], + EncodedCommon::Unused => encoded, + EncodedCommon::Range(range) => &encoded[range.clone()], + EncodedCommon::NoCommonPart => &[], } } // TODO this is bad we should produce a branch that does // not include it in the first place (new encode fn with // default impl using trim no child). - pub fn trim_no_child(self, encoded: &mut Vec) { + // TODO consider removal + pub fn trim_common(self, encoded: &mut Vec) { match self { - EncodedNoChild::Unused => (), - EncodedNoChild::Range(range) => { + EncodedCommon::Unused => (), + EncodedCommon::Range(range) => { encoded.truncate(range.end); if range.start != 0 { *encoded = encoded.split_off(range.start); } }, - EncodedNoChild::Allocated(data) => { - replace(encoded, data); + EncodedCommon::NoCommonPart => { + *encoded = Vec::new(); }, } } @@ -187,7 +189,7 @@ pub trait HashDBComplexDyn: Send + Sync + HashDB { prefix: Prefix, value: &[u8], children: &[Option>], - no_child: EncodedNoChild, + common: EncodedCommon, ) -> H::Out; } @@ -197,7 +199,7 @@ impl> HashDBComplexDyn for C { prefix: Prefix, value: &[u8], children: &[Option>], - no_child: EncodedNoChild, + common: EncodedCommon, ) -> H::Out { // TODO factor this with iter_build (just use the trait) @@ -211,7 +213,7 @@ impl> HashDBComplexDyn for C { self, prefix, value, - no_child.encoded_no_child(value), + common.encoded_common(value), nb_children, children, EmptyIter::default(), diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 04b578af..ca5c48f8 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -120,12 +120,12 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> if !self.is_inline && complex { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - let (mut result, no_child) = C::branch_node_proof( + let (mut result, common) = C::branch_node_common( self.children.iter(), value_with_omission(node_data, value, self.omit_value), register_children, ); - no_child.trim_no_child(&mut result); + common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); @@ -173,14 +173,14 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> // TODO factor with non nibbled!! let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - let (mut result, no_child) = C::branch_node_nibbled_proof( + let (mut result, common) = C::branch_node_nibbled_common( partial.right_iter(), partial.len(), self.children.iter(), value_with_omission(node_data, value, self.omit_value), register_children, ); - no_child.trim_no_child(&mut result); + common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index c9080651..cf845766 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -17,7 +17,7 @@ use crate::rstd::{ }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, EncodedNoChild, + node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, EncodedCommon, }; use hash_db::Hasher; use ordered_trie::{BinaryHasher, HasherComplex}; @@ -149,11 +149,11 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> } /// Encode this entry to an encoded trie node with data properly reconstructed. - fn encode_node(&mut self) -> Result<(Vec, EncodedNoChild), Error> { + fn encode_node(&mut self) -> Result<(Vec, EncodedCommon), Error> { self.complete_children()?; Ok(match self.node { Node::Empty => - (C::empty_node().to_vec(), EncodedNoChild::Unused), + (C::empty_node().to_vec(), EncodedCommon::Unused), Node::Leaf(partial, _) => { let value = self.value .expect( @@ -161,7 +161,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> value is only ever reassigned in the ValueMatch::MatchesLeaf match \ clause, which assigns only to Some" ); - (C::leaf_node(partial.right(), value), EncodedNoChild::Unused) + (C::leaf_node(partial.right(), value), EncodedCommon::Unused) } Node::Extension(partial, _) => { let child = self.children[0] @@ -170,12 +170,12 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> partial.right_iter(), partial.len(), child - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) } Node::Branch(_, _) => { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - C::branch_node_proof( + C::branch_node_common( self.children.iter(), self.value, register_children, // TODO unused register result @@ -184,7 +184,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> Node::NibbledBranch(partial, _, _) => { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); let register_children = &mut register_children[..]; - C::branch_node_nibbled_proof( + C::branch_node_nibbled_common( partial.right_iter(), partial.len(), self.children.iter(), @@ -471,7 +471,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } Step::UnwindStack => { let is_inline = last_entry.is_inline; - let (node_data, no_child) = last_entry.encode_node()?; + let (node_data, common) = last_entry.encode_node()?; let child_ref = if is_inline { if node_data.len() > L::Hash::LENGTH { @@ -501,7 +501,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve }); if let Some(h) = L::Hash::hash_complex( - &no_child.encoded_no_child(node_data.as_slice())[..], + &common.encoded_common(node_data.as_slice())[..], nb_children, children, additional_hash.into_iter(), diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 1b3beb74..87f2c748 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -31,7 +31,7 @@ use crate::{ TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecComplex, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; -use crate::node_codec::{Bitmap, EncodedNoChild}; +use crate::node_codec::{Bitmap, EncodedCommon}; use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, rc::Rc, result, vec, vec::Vec, ops::Range, @@ -126,8 +126,8 @@ impl EncoderStackEntry { } else { (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) }; - let (mut result, no_child) = if let Some(complex) = complex { - C::branch_node_proof( + let (mut result, common) = if let Some(complex) = complex { + C::branch_node_common( children.iter(), value.clone().map(|range| &node_data[range]), complex, @@ -136,10 +136,10 @@ impl EncoderStackEntry { (C::branch_node( children.iter(), value.clone().map(|range| &node_data[range]), - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }; if complex_hash { - no_child.trim_no_child(&mut result); + common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); @@ -178,8 +178,8 @@ impl EncoderStackEntry { (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) }; let partial = partial.build(node_data); - let (mut result, no_child) = if let Some(complex) = complex { - C::branch_node_nibbled_proof( + let (mut result, common) = if let Some(complex) = complex { + C::branch_node_nibbled_common( partial.right_iter(), partial.len(), children.iter(), @@ -192,10 +192,10 @@ impl EncoderStackEntry { partial.len(), children.iter(), value.clone().map(|range| &node_data[range]), - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }; if complex_hash { - no_child.trim_no_child(&mut result); + common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); @@ -474,21 +474,21 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { /// /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. - fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, EncodedNoChild) { + fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, EncodedCommon) { match self.node { Node::Empty => - (C::empty_node().to_vec(), EncodedNoChild::Unused), + (C::empty_node().to_vec(), EncodedCommon::Unused), Node::Leaf(partial, value) => - (C::leaf_node(partial.right(), value), EncodedNoChild::Unused), + (C::leaf_node(partial.right(), value), EncodedCommon::Unused), Node::Extension(partial, _) => (C::extension_node( partial.right_iter(), partial.len(), self.children[0] .expect("required by method precondition; qed"), - ), EncodedNoChild::Unused), + ), EncodedCommon::Unused), Node::Branch(_, value) => if let Some(register_children) = register_children { - C::branch_node_proof( + C::branch_node_common( self.children.into_iter(), value, register_children, @@ -497,10 +497,10 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { (C::branch_node( self.children.into_iter(), value, - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }, Node::NibbledBranch(partial, _, value) => if let Some(register_children) = register_children { - C::branch_node_nibbled_proof( + C::branch_node_nibbled_common( partial.right_iter(), partial.len(), self.children.iter(), @@ -513,7 +513,7 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { partial.len(), self.children.iter(), value, - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }, } } @@ -585,7 +585,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) let complex = last_entry.complex.take(); // Since `advance_child_index` returned true, the preconditions for `encode_node` are // satisfied. - let (node_data, no_child) = last_entry.encode_node(register_children.as_mut().map(|r| r.as_mut())); + let (node_data, common) = last_entry.encode_node(register_children.as_mut().map(|r| r.as_mut())); let node_hash = if let Some((bitmap_keys, additional_hashes)) = complex { let children = register_children.expect("Set to some if complex"); let nb_children = children.iter().filter(|v| v.is_some()).count(); @@ -607,7 +607,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) db.insert_complex( prefix.as_prefix(), &node_data[..], - no_child.encoded_no_child(&node_data[..]), + common.encoded_common(&node_data[..]), nb_children, children, additional_hashes, diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 852b993e..57411829 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -23,7 +23,7 @@ use crate::node_codec::HashDBComplexDyn; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedNoChild}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedCommon}; use crate::nibble::{NibbleVec, NibbleSlice, nibble_ops, BackingByteVec}; use crate::rstd::{ boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, @@ -209,16 +209,16 @@ where self, mut child_cb: F, register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedNoChild) where + ) -> (Vec, EncodedCommon) where C: NodeCodecComplex, F: FnMut(NodeHandle, Option<&NibbleSlice>, Option) -> ChildReference, H: Hasher, { match self { - Node::Empty => (C::empty_node().to_vec(), EncodedNoChild::Unused), + Node::Empty => (C::empty_node().to_vec(), EncodedCommon::Unused), Node::Leaf(partial, value) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - (C::leaf_node(pr.right(), &value), EncodedNoChild::Unused) + (C::leaf_node(pr.right(), &value), EncodedCommon::Unused) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); @@ -228,7 +228,7 @@ where it, pr.len(), c, - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) }, Node::Branch(mut children, value) => { // map the `NodeHandle`s from the Branch to `ChildReferences` @@ -239,7 +239,7 @@ where maybe_child.map(|child| child_cb(child, None, Some(i as u8))) }); if let Some(register_children) = register_children { - C::branch_node_proof( + C::branch_node_common( // map the `NodeHandle`s from the Branch to `ChildReferences` children, value.as_ref().map(|v| &v[..]), @@ -249,7 +249,7 @@ where (C::branch_node( children, value.as_ref().map(|v| &v[..]), - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) } }, Node::NibbledBranch(partial, mut children, value) => { @@ -268,7 +268,7 @@ where }); if let Some(register_children) = register_children { - C::branch_node_nibbled_proof( + C::branch_node_nibbled_common( it, pr.len(), children, @@ -281,7 +281,7 @@ where pr.len(), children, value.as_ref().map(|v| &v[..]), - ), EncodedNoChild::Unused) + ), EncodedCommon::Unused) } }, } From 79f155b1d4ee760c6f91c45f8899af5e765ec47c Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Apr 2020 19:51:18 +0200 Subject: [PATCH 07/73] hash only variant --- test-support/reference-trie/src/lib.rs | 48 +++++++++++++++++++++- trie-db/src/node_codec.rs | 56 ++++++++++++++++++++------ trie-db/src/proof/generate.rs | 12 +----- trie-db/src/trie_codec.rs | 36 +++++++---------- 4 files changed, 106 insertions(+), 46 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 63b1d8f3..f778d5c7 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -804,6 +804,22 @@ impl NodeCodecComplex for ReferenceNodeCodec { ) -> (Vec, EncodedCommon) { unreachable!() } + + fn branch_node_for_hash( + children: impl Iterator>>>, + maybe_value: Option<&[u8]>, + ) -> Vec { + Self::branch_node_internal(children, maybe_value, None, false).0 + } + + fn branch_node_nibbled_for_hash( + _partial: impl Iterator, + _number_nibble: usize, + _children: impl Iterator>>>, + _maybe_value: Option<&[u8]>, + ) -> Vec { + unreachable!() + } } impl ReferenceNodeCodec { @@ -1164,7 +1180,37 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { maybe_value: Option<&[u8]>, register_children: &mut [Option>], ) -> (Vec, EncodedCommon) { - Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, Some(register_children), true) + Self::branch_node_nibbled_internal( + partial, + number_nibble, + children, + maybe_value, + Some(register_children), + true, + ) + } + + fn branch_node_for_hash( + _children: impl Iterator::Out>>>>, + _maybe_value: Option<&[u8]>, + ) -> Vec { + unreachable!() + } + + fn branch_node_nibbled_for_hash( + partial: impl Iterator, + number_nibble: usize, + children: impl Iterator>>>, + maybe_value: Option<&[u8]>, + ) -> Vec { + Self::branch_node_nibbled_internal( + partial, + number_nibble, + children, + maybe_value, + None, + false, + ).0 } } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index a71b6387..4401bacb 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -113,19 +113,20 @@ pub trait NodeCodecComplex: NodeCodec { Ok((plan.build(data), hashes)) } - /// Returns an encoded branch node, and additional information for getting the common - /// encoded part with the proof base encoding. + /// Returns branch node encoded for storage, and additional information for hash calculation. /// - /// Takes an iterator yielding `ChildReference` and an optional value. + /// Takes an iterator yielding `ChildReference` and an optional value + /// as input, the third input is an output container needed for hash calculation. fn branch_node_common( children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], ) -> (Vec, EncodedCommon); - /// Returns an encoded branch node with a possible partial path, and additional information - /// for getting the encoding without child common part. - /// `number_nibble` is the partial path length as in `extension_node`. + /// Variant of `branch_node_common` but with a nibble. + /// + /// `number_nibble` is the partial path length, it replaces the one + /// use by `extension_node`. fn branch_node_nibbled_common( partial: impl Iterator, number_nibble: usize, @@ -133,16 +134,45 @@ pub trait NodeCodecComplex: NodeCodec { value: Option<&[u8]>, register_children: &mut [Option>], ) -> (Vec, EncodedCommon); + + /// Returns branch node encoded information for hash. + /// Result is the same as `branch_node_common().1.encoded_common(branch_node_common().0`. + fn branch_node_for_hash( + children: impl Iterator>>>, + value: Option<&[u8]>, + ) -> Vec; + + /// Variant of `branch_node_for_hash` but with a nibble. + fn branch_node_nibbled_for_hash( + partial: impl Iterator, + number_nibble: usize, + children: impl Iterator>>>, + value: Option<&[u8]>, + ) -> Vec; + + /// Build compact proof encoding from branch info. + fn proof_compact_encoded( + branch_basis: Vec, + number_nibble: usize, + children: impl Iterator>>>, + value: Option<&[u8]>, + register_children: &[Option>], + ) -> Vec { + unimplemented!("TODO factor this code!!(look for _for_hash current calls)") + } } +/// Information to fetch bytes that needs to be include when calculating a node hash. +/// The node hash is the hash of these information and the merkle root of its children. +/// TODO EMCH rename to BranchHashInfo #[derive(Clone)] pub enum EncodedCommon { - // not runing complex + /// No need for complex hash. TODO EMCH see if still used. Unused, - // range over the encoded common part with the storage encoded + /// Range over the branch encoded for storage. Range(Range), - // allocated in case we cannot use a range - NoCommonPart, + /// Allocated in case we cannot use a range. + Allocated(Vec), } impl EncodedCommon { @@ -150,7 +180,7 @@ impl EncodedCommon { match self { EncodedCommon::Unused => encoded, EncodedCommon::Range(range) => &encoded[range.clone()], - EncodedCommon::NoCommonPart => &[], + EncodedCommon::Allocated(buff) => &buff[..], } } // TODO this is bad we should produce a branch that does @@ -166,8 +196,8 @@ impl EncodedCommon { *encoded = encoded.split_off(range.start); } }, - EncodedCommon::NoCommonPart => { - *encoded = Vec::new(); + EncodedCommon::Allocated(buf) => { + *encoded = buf; }, } } diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index ca5c48f8..72b6c46a 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -118,14 +118,10 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> &mut self.children, )?; if !self.is_inline && complex { - let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); - let register_children = &mut register_children[..]; - let (mut result, common) = C::branch_node_common( + let mut result = C::branch_node_for_hash( self.children.iter(), value_with_omission(node_data, value, self.omit_value), - register_children, ); - common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); @@ -171,16 +167,12 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> )?; if !self.is_inline && complex { // TODO factor with non nibbled!! - let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); - let register_children = &mut register_children[..]; - let (mut result, common) = C::branch_node_nibbled_common( + let mut result = C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), self.children.iter(), value_with_omission(node_data, value, self.omit_value), - register_children, ); - common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 87f2c748..8eb231db 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -118,28 +118,24 @@ impl EncoderStackEntry { } } NodePlan::Branch { value, children } => { - let mut register: [Option<_>; NIBBLE_LENGTH]; // TODO unused register - let (children, complex) = if complex_hash { + let children = if complex_hash { let no_omit = [false; NIBBLE_LENGTH]; - register = Default::default(); - (Self::branch_children(node_data, &children, &no_omit[..])?, Some(&mut register[..])) + Self::branch_children(node_data, &children, &no_omit[..])? } else { - (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) + Self::branch_children(node_data, &children, &self.omit_children[..])? }; - let (mut result, common) = if let Some(complex) = complex { - C::branch_node_common( + let mut result = if complex_hash { + C::branch_node_for_hash( children.iter(), value.clone().map(|range| &node_data[range]), - complex, ) } else { - (C::branch_node( + C::branch_node( children.iter(), value.clone().map(|range| &node_data[range]), - ), EncodedCommon::Unused) + ) }; if complex_hash { - common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); @@ -169,33 +165,29 @@ impl EncoderStackEntry { result } NodePlan::NibbledBranch { partial, value, children } => { - let mut register: [Option<_>; NIBBLE_LENGTH]; // TODO unused register - let (children, complex) = if complex_hash { + let children = if complex_hash { let no_omit = [false; NIBBLE_LENGTH]; - register = Default::default(); - (Self::branch_children(node_data, &children, &no_omit[..])?, Some(&mut register[..])) + Self::branch_children(node_data, &children, &no_omit[..])? } else { - (Self::branch_children(node_data, &children, &self.omit_children[..])?, None) + Self::branch_children(node_data, &children, &self.omit_children[..])? }; let partial = partial.build(node_data); - let (mut result, common) = if let Some(complex) = complex { - C::branch_node_nibbled_common( + let mut result = if complex_hash { + C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), children.iter(), value.clone().map(|range| &node_data[range]), - complex, ) } else { - (C::branch_node_nibbled( + C::branch_node_nibbled( partial.right_iter(), partial.len(), children.iter(), value.clone().map(|range| &node_data[range]), - ), EncodedCommon::Unused) + ) }; if complex_hash { - common.trim_common(&mut result); let bitmap_start = result.len(); result.push(0u8); result.push(0u8); From cf1e95f037005bf6a5b8dee1323011a5953d9505 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 10:57:35 +0200 Subject: [PATCH 08/73] codec encode function --- test-support/reference-trie/src/lib.rs | 63 ++++++++++++++++++++++++++ trie-db/src/lib.rs | 2 +- trie-db/src/node_codec.rs | 31 +++++++++---- trie-db/src/proof/generate.rs | 14 +++--- trie-db/src/trie_codec.rs | 8 +++- 5 files changed, 99 insertions(+), 19 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index f778d5c7..098ee4ab 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -32,6 +32,7 @@ use trie_db::{ BinaryHasher, EncodedCommon, HashesPlan, + binary_additional_hashes, }; use std::borrow::Borrow; use keccak_hasher::KeccakHasher; @@ -820,6 +821,16 @@ impl NodeCodecComplex for ReferenceNodeCodec { ) -> Vec { unreachable!() } + + fn encode_compact_proof( + hash_proof_header: Vec, + in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], + children: &[Option>], + hash_buf: &mut BH::Buffer, + ) -> Vec { + encode_proof_internal::(hash_proof_header, in_proof_children, children, hash_buf) + } + } impl ReferenceNodeCodec { @@ -945,6 +956,48 @@ fn decode_plan_proof_internal( Ok((node, hashes_plan)) } +fn encode_proof_internal( + mut result: Vec, + mut in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], + children: &[Option>], + hash_buf: &mut H::Buffer, +) -> Vec { + let bitmap_start = result.len(); + result.push(0u8); + result.push(0u8); + // write all inline nodes TODO we could omit children first + // as in std case and fill this bitmap as in generate.rs. + for (ix, child) in children.iter().enumerate() { + // TODO EMCH seems like we do not need in_proof_children input + // How does it differs from standard bitmap?? + if let Some(ChildReference::Inline(h, nb)) = child.borrow() { + // TODO do not write inline of null size, these are defined + // in the bitmap and from the algos. + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + in_proof_children[ix] = true; + } + } + // We write a bitmap containing all children node that are either ommitted from the + // proof or inline nodes encoded in the proof. + // TODO seems useless as we build it from an empty input (maybe not for trie_codec + // not sure if ommited node are children of trie codec already). + Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); + + let additional_hashes = binary_additional_hashes::( + &children[..], + &in_proof_children[..], + hash_buf, + ); + result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes + for hash in additional_hashes { + result.extend_from_slice(hash.as_ref()); + } + result +} + impl ReferenceNodeCodecNoExt { fn decode_plan_internal( data: &[u8], @@ -1212,6 +1265,16 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { false, ).0 } + + fn encode_compact_proof( + hash_proof_header: Vec, + in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], + children: &[Option>], + hash_buf: &mut BH::Buffer, + ) -> Vec { + encode_proof_internal::(hash_proof_header, in_proof_children, children, hash_buf) + } + } /// Compare trie builder and in memory trie. diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 742890fd..88c59aa8 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -75,7 +75,7 @@ pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexD pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; pub use crate::iterator::TrieDBNodeIterator; -pub use crate::trie_codec::{decode_compact, encode_compact}; +pub use crate::trie_codec::{decode_compact, encode_compact, binary_additional_hashes}; pub use ordered_trie::BinaryHasher; #[cfg(feature = "std")] diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 4401bacb..51b695da 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -18,6 +18,8 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; +use crate::nibble::nibble_ops; +use ordered_trie::BinaryHasher; use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData}; @@ -97,6 +99,11 @@ pub trait NodeCodecComplex: NodeCodec { /// TODO EMCH this is technical function for common implementation. /// TODO document this damn bitmap!!! + /// The parameter bitmap indicates children that are directly encoded + /// in the proof or encoded as inline. That is the difference between + /// the set of children and the set of children that are part of the + /// additional hash of the proof. TODO remove it (not needed if we + /// remove it from encode input, then doc that at codec level). fn decode_plan_proof(data: &[u8]) -> Result<( NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>, @@ -151,15 +158,21 @@ pub trait NodeCodecComplex: NodeCodec { ) -> Vec; /// Build compact proof encoding from branch info. - fn proof_compact_encoded( - branch_basis: Vec, - number_nibble: usize, - children: impl Iterator>>>, - value: Option<&[u8]>, - register_children: &[Option>], - ) -> Vec { - unimplemented!("TODO factor this code!!(look for _for_hash current calls)") - } + /// + /// - `hash_proof_header`: the part common with the header info from hash. + /// It can be calculated from `branch_node_common` through + /// `EncodedCommon` call, or directly by `branch_node_for_hash`. + /// TODO EMCH rename this common `HashProofHeader`. + /// - `in_proof_children`: presence of children in the proof, this is typically + /// ommitted children from compact proof encoded as a 0 length children. These + /// will not be encoded as hash for child proof. + /// TODO consider using bitmap directly + fn encode_compact_proof( + hash_proof_header: Vec, + in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], + children: &[Option>], + hash_buf: &mut H::Buffer, + ) -> Vec; } /// Information to fetch bytes that needs to be include when calculating a node hash. diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 72b6c46a..b28ae914 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -130,19 +130,17 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> // TODO again register for nothing for (ix, child) in self.children.iter().enumerate() { if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { - if *nb > 0 { - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); - } - in_proof_children[ix] = true; + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + in_proof_children[ix] = true; } } Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); let additional_hashes = crate::trie_codec::binary_additional_hashes::( &self.children[..], - &in_proof_children[..], + &in_proof_children[..], hash_buf, ); result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 8eb231db..cc5ffe00 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -624,7 +624,13 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) Err(Box::new(TrieError::IncompleteDatabase(>::default()))) } -pub(crate) fn binary_additional_hashes( +/// Returns the additional hashes requires for a proof. +/// - `children`, the children for a branch +/// - `in_proof_children`, for all children, +/// indicates if it is included in the proof (inline node or +/// compacted node). +/// - `hash_buf` a buffer of the right size to compute the hash. +pub fn binary_additional_hashes( children: &[Option>], in_proof_children: &[bool], hash_buf: &mut H::Buffer, From 02b149e852905718b8c02d625beeb5644fa38bfa Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 11:07:41 +0200 Subject: [PATCH 09/73] use new encode_proof method --- trie-db/src/proof/generate.rs | 64 +++++---------------------- trie-db/src/trie_codec.rs | 83 +++++++++-------------------------- 2 files changed, 31 insertions(+), 116 deletions(-) diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index b28ae914..11664710 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -20,7 +20,6 @@ use crate::rstd::{ use hash_db::Hasher; -use crate::node_codec::Bitmap; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, Result as TrieResult, Trie, TrieError, TrieHash, @@ -118,36 +117,16 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> &mut self.children, )?; if !self.is_inline && complex { - let mut result = C::branch_node_for_hash( + let hash_proof_header = C::branch_node_for_hash( self.children.iter(), value_with_omission(node_data, value, self.omit_value), ); - let bitmap_start = result.len(); - result.push(0u8); - result.push(0u8); - let mut in_proof_children = [false; NIBBLE_LENGTH]; - // write all inline nodes and ommitted node - // TODO again register for nothing - for (ix, child) in self.children.iter().enumerate() { - if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); - in_proof_children[ix] = true; - } - } - Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); - let additional_hashes = crate::trie_codec::binary_additional_hashes::( + C::encode_compact_proof::( + hash_proof_header, + [false; NIBBLE_LENGTH], &self.children[..], - &in_proof_children[..], hash_buf, - ); - result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes - for hash in additional_hashes { - result.extend_from_slice(hash.as_ref()); - } - result + ) } else { C::branch_node( self.children.into_iter(), @@ -164,41 +143,18 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> &mut self.children )?; if !self.is_inline && complex { - // TODO factor with non nibbled!! - let mut result = C::branch_node_nibbled_for_hash( + let hash_proof_header = C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), self.children.iter(), value_with_omission(node_data, value, self.omit_value), ); - let bitmap_start = result.len(); - result.push(0u8); - result.push(0u8); - let mut in_proof_children = [false; NIBBLE_LENGTH]; - // write all inline nodes and ommitted node - // TODO again register for nothing - for (ix, child) in self.children.iter().enumerate() { - if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { - if *nb > 0 { - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); - } - in_proof_children[ix] = true; - } - } - Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); - let additional_hashes = crate::trie_codec::binary_additional_hashes::( + C::encode_compact_proof::( + hash_proof_header, + [false; NIBBLE_LENGTH], &self.children[..], - &in_proof_children[..], hash_buf, - ); - result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes - for hash in additional_hashes { - result.extend_from_slice(hash.as_ref()); - } - result + ) } else { C::branch_node_nibbled( partial.right_iter(), diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index cc5ffe00..32bf238d 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -116,7 +116,7 @@ impl EncoderStackEntry { let empty_child = ChildReference::Inline(C::HashOut::default(), 0); C::extension_node(partial.right_iter(), partial.len(), empty_child) } - } + }, NodePlan::Branch { value, children } => { let children = if complex_hash { let no_omit = [false; NIBBLE_LENGTH]; @@ -124,46 +124,25 @@ impl EncoderStackEntry { } else { Self::branch_children(node_data, &children, &self.omit_children[..])? }; - let mut result = if complex_hash { - C::branch_node_for_hash( + if complex_hash { + let hash_proof_header = C::branch_node_for_hash( children.iter(), value.clone().map(|range| &node_data[range]), + ); + let in_proof_children = self.omit_children.clone(); + C::encode_compact_proof::( + hash_proof_header, + in_proof_children, + &children[..], + hash_buf, ) } else { C::branch_node( children.iter(), value.clone().map(|range| &node_data[range]), ) - }; - if complex_hash { - let bitmap_start = result.len(); - result.push(0u8); - result.push(0u8); - let mut in_proof_children = self.omit_children.clone(); - // write all inline nodes TODO we could omit children first - // as in std case and fill this bitmap as in generate.rs. - for (ix, child) in children.iter().enumerate() { - if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); - in_proof_children[ix] = true; - } - } - Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); - let additional_hashes = binary_additional_hashes::( - &children[..], - &in_proof_children[..], - hash_buf, - ); - result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes - for hash in additional_hashes { - result.extend_from_slice(hash.as_ref()); - } } - result - } + }, NodePlan::NibbledBranch { partial, value, children } => { let children = if complex_hash { let no_omit = [false; NIBBLE_LENGTH]; @@ -172,12 +151,19 @@ impl EncoderStackEntry { Self::branch_children(node_data, &children, &self.omit_children[..])? }; let partial = partial.build(node_data); - let mut result = if complex_hash { - C::branch_node_nibbled_for_hash( + if complex_hash { + let hash_proof_header = C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), children.iter(), value.clone().map(|range| &node_data[range]), + ); + let in_proof_children = self.omit_children.clone(); + C::encode_compact_proof::( + hash_proof_header, + in_proof_children, + &children[..], + hash_buf, ) } else { C::branch_node_nibbled( @@ -186,35 +172,8 @@ impl EncoderStackEntry { children.iter(), value.clone().map(|range| &node_data[range]), ) - }; - if complex_hash { - let bitmap_start = result.len(); - result.push(0u8); - result.push(0u8); - let mut in_proof_children = self.omit_children.clone(); - // write all inline nodes - for (ix, child) in children.iter().enumerate() { - if let Some(ChildReference::Inline(h, nb)) = child.as_ref() { - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); - in_proof_children[ix] = true; - } - } - Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); - let additional_hashes = binary_additional_hashes::( - &children[..], - &in_proof_children[..], - hash_buf, - ); - result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes - for hash in additional_hashes { - result.extend_from_slice(hash.as_ref()); - } } - result - } + }, }) } From 1aeeb7900c44c69236d8bba8f924cf799e0480f3 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 11:34:01 +0200 Subject: [PATCH 10/73] remove unneeded param and do not write ommited children in proof --- test-support/reference-trie/src/lib.rs | 22 +++++++++++----------- trie-db/src/node_codec.rs | 7 ++----- trie-db/src/proof/generate.rs | 2 -- trie-db/src/trie_codec.rs | 18 ++---------------- 4 files changed, 15 insertions(+), 34 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 098ee4ab..efb86669 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -824,11 +824,10 @@ impl NodeCodecComplex for ReferenceNodeCodec { fn encode_compact_proof( hash_proof_header: Vec, - in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], children: &[Option>], hash_buf: &mut BH::Buffer, ) -> Vec { - encode_proof_internal::(hash_proof_header, in_proof_children, children, hash_buf) + encode_proof_internal::(hash_proof_header, children, hash_buf) } } @@ -958,10 +957,10 @@ fn decode_plan_proof_internal( fn encode_proof_internal( mut result: Vec, - mut in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], children: &[Option>], hash_buf: &mut H::Buffer, ) -> Vec { + let mut in_proof_children = [false; nibble_ops::NIBBLE_LENGTH]; let bitmap_start = result.len(); result.push(0u8); result.push(0u8); @@ -971,12 +970,14 @@ fn encode_proof_internal( // TODO EMCH seems like we do not need in_proof_children input // How does it differs from standard bitmap?? if let Some(ChildReference::Inline(h, nb)) = child.borrow() { - // TODO do not write inline of null size, these are defined - // in the bitmap and from the algos. - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); + if *nb > 0 { + // TODO do not write inline of null size, these are defined + // in the bitmap and from the algos. + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + } in_proof_children[ix] = true; } } @@ -1268,11 +1269,10 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { fn encode_compact_proof( hash_proof_header: Vec, - in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], children: &[Option>], hash_buf: &mut BH::Buffer, ) -> Vec { - encode_proof_internal::(hash_proof_header, in_proof_children, children, hash_buf) + encode_proof_internal::(hash_proof_header, children, hash_buf) } } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 51b695da..eee693ba 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -18,7 +18,6 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use crate::nibble::nibble_ops; use ordered_trie::BinaryHasher; use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData}; @@ -163,13 +162,11 @@ pub trait NodeCodecComplex: NodeCodec { /// It can be calculated from `branch_node_common` through /// `EncodedCommon` call, or directly by `branch_node_for_hash`. /// TODO EMCH rename this common `HashProofHeader`. - /// - `in_proof_children`: presence of children in the proof, this is typically - /// ommitted children from compact proof encoded as a 0 length children. These - /// will not be encoded as hash for child proof. + /// - `children`: contains all children reference, not that children reference + /// that are compacted are set as inline children of length 0. /// TODO consider using bitmap directly fn encode_compact_proof( hash_proof_header: Vec, - in_proof_children: [bool; nibble_ops::NIBBLE_LENGTH], children: &[Option>], hash_buf: &mut H::Buffer, ) -> Vec; diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 11664710..bd53fb15 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -123,7 +123,6 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> ); C::encode_compact_proof::( hash_proof_header, - [false; NIBBLE_LENGTH], &self.children[..], hash_buf, ) @@ -151,7 +150,6 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> ); C::encode_compact_proof::( hash_proof_header, - [false; NIBBLE_LENGTH], &self.children[..], hash_buf, ) diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 32bf238d..804fc9be 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -118,21 +118,14 @@ impl EncoderStackEntry { } }, NodePlan::Branch { value, children } => { - let children = if complex_hash { - let no_omit = [false; NIBBLE_LENGTH]; - Self::branch_children(node_data, &children, &no_omit[..])? - } else { - Self::branch_children(node_data, &children, &self.omit_children[..])? - }; + let children = Self::branch_children(node_data, &children, &self.omit_children[..])?; if complex_hash { let hash_proof_header = C::branch_node_for_hash( children.iter(), value.clone().map(|range| &node_data[range]), ); - let in_proof_children = self.omit_children.clone(); C::encode_compact_proof::( hash_proof_header, - in_proof_children, &children[..], hash_buf, ) @@ -144,12 +137,7 @@ impl EncoderStackEntry { } }, NodePlan::NibbledBranch { partial, value, children } => { - let children = if complex_hash { - let no_omit = [false; NIBBLE_LENGTH]; - Self::branch_children(node_data, &children, &no_omit[..])? - } else { - Self::branch_children(node_data, &children, &self.omit_children[..])? - }; + let children = Self::branch_children(node_data, &children, &self.omit_children[..])?; let partial = partial.build(node_data); if complex_hash { let hash_proof_header = C::branch_node_nibbled_for_hash( @@ -158,10 +146,8 @@ impl EncoderStackEntry { children.iter(), value.clone().map(|range| &node_data[range]), ); - let in_proof_children = self.omit_children.clone(); C::encode_compact_proof::( hash_proof_header, - in_proof_children, &children[..], hash_buf, ) From 7762b8d4154047e595072dc4345789cf5ecfd939 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 12:57:21 +0200 Subject: [PATCH 11/73] wrong approach, we are not including ommitted node to the proof. --- trie-db/src/node_codec.rs | 14 ++++++-------- trie-db/src/proof/verify.rs | 3 +++ trie-db/src/trie_codec.rs | 20 ++++++++++---------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index eee693ba..d8a73b7a 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -97,12 +97,9 @@ pub trait NodeCodecComplex: NodeCodec { type AdditionalHashesPlan: Iterator>; /// TODO EMCH this is technical function for common implementation. - /// TODO document this damn bitmap!!! - /// The parameter bitmap indicates children that are directly encoded - /// in the proof or encoded as inline. That is the difference between - /// the set of children and the set of children that are part of the - /// additional hash of the proof. TODO remove it (not needed if we - /// remove it from encode input, then doc that at codec level). + /// The parameter bitmap indicates children that are ommitted because + /// they are not needed by the hashing or because they are compacted. + /// The additional hashes are the hashes required to verify the proof. fn decode_plan_proof(data: &[u8]) -> Result<( NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>, @@ -162,8 +159,9 @@ pub trait NodeCodecComplex: NodeCodec { /// It can be calculated from `branch_node_common` through /// `EncodedCommon` call, or directly by `branch_node_for_hash`. /// TODO EMCH rename this common `HashProofHeader`. - /// - `children`: contains all children reference, not that children reference - /// that are compacted are set as inline children of length 0. + /// - `children`: contains all children reference to include in the proof, + /// note that children reference that are compacted are set as inline children of length 0. + /// /// TODO consider using bitmap directly fn encode_compact_proof( hash_proof_header: Vec, diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index cf845766..d9be95e7 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -490,6 +490,9 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve v.as_ref().map(|v| (ix, v.clone())) }) .map(|(ix, child_ref)| { + // Use of bitmap_keys here to avoid + // adding a reference that is ommitted + // from the proof. if bitmap_keys.value_at(ix) { Some(match child_ref { ChildReference::Hash(h) => h, diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 804fc9be..53804209 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -331,10 +331,13 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { self.child_index += 1; } Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { - if let Some((bitmap, _)) = self.complex.as_ref() { + if let Some((bitmap_keys, _)) = self.complex.as_ref() { while self.child_index < NIBBLE_LENGTH { match children[self.child_index] { - Some(NodeHandle::Inline(data)) if data.is_empty() && bitmap.value_at(self.child_index) => { + Some(NodeHandle::Inline(data)) if data.is_empty() + // Use of bitmap_keys here to avoid going into + // a child that is ommitted from the binary hash proof. + && bitmap_keys.value_at(self.child_index) => { return Ok(false); }, Some(child) => { @@ -523,18 +526,15 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) // Since `advance_child_index` returned true, the preconditions for `encode_node` are // satisfied. let (node_data, common) = last_entry.encode_node(register_children.as_mut().map(|r| r.as_mut())); - let node_hash = if let Some((bitmap_keys, additional_hashes)) = complex { + let node_hash = if let Some((_bitmap_keys, additional_hashes)) = complex { let children = register_children.expect("Set to some if complex"); let nb_children = children.iter().filter(|v| v.is_some()).count(); let children = children.iter() - .enumerate() - .filter_map(|(ix, v)| { - v.as_ref().map(|v| (ix, v.clone())) - }) - .map(|(ix, range)| { - if bitmap_keys.value_at(ix) { + .filter_map(|v| v.clone()) + .map(|range| { + let len = range.end - range.start; + if len > 0 { let mut v = TrieHash::::default(); - let len = range.end - range.start; v.as_mut()[..len].copy_from_slice(&node_data[range]); Some(v) } else { From 2c66cc293d3bb24c7371168bce9131c9d1fe8ca8 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 16:18:14 +0200 Subject: [PATCH 12/73] skip some hash round from HashRoot, still missing some cases. --- ordered-trie/src/lib.rs | 46 ++++++++++++++++++++------ test-support/reference-trie/src/lib.rs | 8 ++--- trie-db/src/node_codec.rs | 24 +++++++++----- trie-db/src/trie_codec.rs | 14 ++++++-- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 3e2289ec..2d738d2f 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -286,7 +286,7 @@ impl SequenceBinaryTree { } } - /// resolve the tree path for a given index. + /// Resolve the tree path for a given index. pub fn path_node_key>(&self, index: usize) -> KN { let tmp = (!0usize << (self.depth - self.end_depth)) | (index >> self.end_depth); if !tmp == 0 { @@ -798,13 +798,16 @@ impl<'a, H: BinaryHasher, KN> ProcessNode for HashOnly<'a, H> { /// Buffer length need to be right and is unchecked, proof elements are /// stored in memory (no streaming). -pub struct HashProof<'a, H: Hasher, I, KN>{ +pub struct HashProof<'a, H: Hasher, I, KN> { buffer: &'a mut [u8], - // I must guaranty right depth and index in range regarding - // to the tree struct!!! + // `I` is a iterator over the depth of every element included in the proof. + // Can be obtain from `iter_path_node_key` filtered by contained elements. to_prove: I, state: MultiProofState, additional_hash: Vec, + // when we do not need to calculate the root but just the additional hashes + // we can set it to true to avoid useless hashes round. + additional_hash_only: bool, } // We need some read ahead to manage state. @@ -951,13 +954,18 @@ impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> HashProof<'a, H, // TODO write function to build from iter of unchecked usize indexes: map iter // with either depth_at or the depth_iterator skipping undesired elements (second // seems better as it filters out of range. - pub fn new(buff: &'a mut H::Buffer, mut to_prove: I) -> Self { + pub fn new( + buff: &'a mut H::Buffer, + mut to_prove: I, + additional_hash_only: bool, + ) -> Self { let state = MultiProofState::new(&mut to_prove); HashProof { buffer: buff.as_mut(), to_prove, state, additional_hash: Vec::new(), + additional_hash_only, } } pub fn take_additional_hash(&mut self) -> Vec { @@ -970,23 +978,30 @@ impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> ProcessNode H::Out { + let mut skip_hashing = false;; match self.state.new_key(key, &mut self.to_prove) { MultiProofResult::DoNothing => (), MultiProofResult::RegisterLeft => { let mut to_push = H::Out::default(); to_push.as_mut().copy_from_slice(child1); self.additional_hash.push(to_push); + skip_hashing = self.additional_hash_only; }, MultiProofResult::RegisterRight => { let mut to_push = H::Out::default(); to_push.as_mut().copy_from_slice(child2); self.additional_hash.push(to_push); + skip_hashing = self.additional_hash_only; }, } - self.buffer[..H::LENGTH].copy_from_slice(child1); - self.buffer[H::LENGTH..].copy_from_slice(child2); - H::hash(&self.buffer[..]) + if skip_hashing { + Default::default() + } else { + self.buffer[..H::LENGTH].copy_from_slice(child1); + self.buffer[H::LENGTH..].copy_from_slice(child2); + H::hash(&self.buffer[..]) + } } fn register_root(&mut self, root: &H::Out) { if self.state.is_empty { @@ -1394,14 +1409,20 @@ mod test { let tree = Tree::new(0, 0, l); let mut hash_buf = ::Buffer::default(); let mut hash_buf2 = ::Buffer::default(); + let mut hash_buf3 = ::Buffer::default(); let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); let hashes: Vec<_> = hashes(l); for p in 0..l { let to_prove = vec![tree.path_node_key::(p)]; - let mut callback = HashProof::::new(&mut hash_buf, to_prove.into_iter()); + let mut callback = HashProof::::new(&mut hash_buf, to_prove.into_iter(), false); + let to_prove = vec![tree.path_node_key::(p)]; let root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); + let mut callback2 = HashProof::::new(&mut hash_buf3, to_prove.into_iter(), true); + let _ = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback2); assert_eq!(root.as_ref(), &result[l][..]); let additional_hash = callback.take_additional_hash(); + let additional_hash_2 = callback2.take_additional_hash(); + assert_eq!(additional_hash, additional_hash_2); let proof_items = vec![(tree.path_node_key::(p), hashes[p].clone())]; let root = trie_root_from_proof::<_, UsizeKeyNode, _, _, _>( &tree, @@ -1442,6 +1463,7 @@ mod test { let tree = Tree::new(0, 0, l); let mut hash_buf = ::Buffer::default(); let mut hash_buf2 = ::Buffer::default(); + let mut hash_buf3 = ::Buffer::default(); let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); let hashes: Vec<_> = hashes(l); let mut to_prove = Vec::new(); @@ -1451,10 +1473,14 @@ mod test { to_prove.push(tree.path_node_key::(p)); proof_items.push((tree.path_node_key::(p), hashes[p].clone())); } - let mut callback = HashProof::::new(&mut hash_buf, to_prove.into_iter()); + let mut callback = HashProof::::new(&mut hash_buf, to_prove.clone().into_iter(), false); + let mut callback2 = HashProof::::new(&mut hash_buf3, to_prove.into_iter(), true); let root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); + let _ = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback2); assert_eq!(root.as_ref(), &result[l][..]); let additional_hash = callback.take_additional_hash(); + let additional_hash2 = callback2.take_additional_hash(); + assert_eq!(additional_hash, additional_hash2); let root = trie_root_from_proof::<_, UsizeKeyNode, _, _, _>( &tree, proof_items, diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index efb86669..dc9fac6f 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -918,7 +918,6 @@ fn decode_plan_proof_internal( if data.len() < offset + 3 { return Err(CodecError::from("Decode branch, missing proof headers")); } - // TODO EMCH bitmap looks unused!!! let keys_position = Bitmap::decode(&data[offset..offset + BITMAP_LENGTH]); offset += BITMAP_LENGTH; @@ -981,10 +980,9 @@ fn encode_proof_internal( in_proof_children[ix] = true; } } - // We write a bitmap containing all children node that are either ommitted from the - // proof or inline nodes encoded in the proof. - // TODO seems useless as we build it from an empty input (maybe not for trie_codec - // not sure if ommited node are children of trie codec already). + // We write a bitmap containing all children node that are included in the binary + // child proof construction. + // In practice, that is inline values and ommited compacted values). Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); let additional_hashes = binary_additional_hashes::( diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index d8a73b7a..5d73447a 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -96,17 +96,21 @@ pub trait NodeCodecComplex: NodeCodec { /// Sequence of hashes needed for the children proof verification. type AdditionalHashesPlan: Iterator>; - /// TODO EMCH this is technical function for common implementation. - /// The parameter bitmap indicates children that are ommitted because - /// they are not needed by the hashing or because they are compacted. - /// The additional hashes are the hashes required to verify the proof. + /// Technical function to implement `decode_proof`. fn decode_plan_proof(data: &[u8]) -> Result<( NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>, ), Self::Error>; - /// Decode but child are not include (instead we put empty inline - /// nodes). + /// Decode a `Node` from a proof. The node can miss some branch that + /// are not use by the proof, in this case they are stored as an inline + /// zero length child. + /// + /// Accompanying resulting node, is also a bitmap of children included in + /// the proof calculation and a sequence of additianal node for proof + /// verification. + /// In practice this contains inline node (inline nodes are always added + /// to the proof) and ommitted children hash (compacted hash). fn decode_proof(data: &[u8]) -> Result<( Node, Option<(Bitmap, HashesIter)>, @@ -159,8 +163,12 @@ pub trait NodeCodecComplex: NodeCodec { /// It can be calculated from `branch_node_common` through /// `EncodedCommon` call, or directly by `branch_node_for_hash`. /// TODO EMCH rename this common `HashProofHeader`. - /// - `children`: contains all children reference to include in the proof, - /// note that children reference that are compacted are set as inline children of length 0. + /// - `children`: contains all children, with compact (ommited children) defined as + /// a null length inline node. + /// The children to be include in the proof are therefore the compacted one and the + /// inline nodes only. + /// The other children value are needed because they can be included into the additional + /// hash, and are required for intermediate hash calculation. /// /// TODO consider using bitmap directly fn encode_compact_proof( diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 53804209..360e5697 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -533,6 +533,9 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) .filter_map(|v| v.clone()) .map(|range| { let len = range.end - range.start; + // We know that these are not ommitted node, they are + // rebuild at this point (otherwhise we could query + // bitmap_keys to ensure that). if len > 0 { let mut v = TrieHash::::default(); v.as_mut()[..len].copy_from_slice(&node_data[range]); @@ -575,6 +578,13 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) /// indicates if it is included in the proof (inline node or /// compacted node). /// - `hash_buf` a buffer of the right size to compute the hash. +/// +/// TODO EMCH this can be highly optimized: we do not need to calculate +/// hash up to the root but only up to the needed intermediate hash. +/// -> custom callback different than trie_root probably +/// at this time trie_root run on empty inline node and produce some incorrect +/// path that cannot be included in additional hash. +/// TODO and can move to ordered_trie?? pub fn binary_additional_hashes( children: &[Option>], in_proof_children: &[bool], @@ -594,14 +604,14 @@ pub fn binary_additional_hashes( }); - let mut callback = HashProof::::new(hash_buf, to_prove); + let mut callback = HashProof::::new(hash_buf, to_prove, true); let hashes = children.iter() .filter_map(|o_child| o_child.as_ref()) .map(|child| match child { ChildReference::Hash(h) => h.clone(), ChildReference::Inline(h, _) => h.clone(), }); - // TODO we can skip a hash (the one to calculate root) + // TODO we can skip a hash (the one to calculate root), actually we can skip all hash let _root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); callback.take_additional_hash() } From ecc85131a42ebc78cdf6ab8a6d2ddd9ddf0e5c38 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 16:34:19 +0200 Subject: [PATCH 13/73] correct skip of round, if key is start of one of target key we can skip. --- ordered-trie/src/lib.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 2d738d2f..e30ffac1 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -827,6 +827,7 @@ enum MultiProofResult { RegisterLeft, RegisterRight, DoNothing, + DoNothingNoStartWith, } impl MultiProofState { @@ -946,7 +947,7 @@ impl MultiProofState { } } } - MultiProofResult::DoNothing + MultiProofResult::DoNothingNoStartWith } } @@ -978,9 +979,12 @@ impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> ProcessNode H::Out { - let mut skip_hashing = false;; + let mut skip_hashing = false; match self.state.new_key(key, &mut self.to_prove) { - MultiProofResult::DoNothing => (), + MultiProofResult::DoNothingNoStartWith => (), + MultiProofResult::DoNothing => { + skip_hashing = self.additional_hash_only + }, MultiProofResult::RegisterLeft => { let mut to_push = H::Out::default(); to_push.as_mut().copy_from_slice(child1); From 037db169d688b53805007ca3339d249091e53897 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 17:09:08 +0200 Subject: [PATCH 14/73] fix and small change, generally new_key need rewrite (the second read ahead logic is based on the first one which makes thing a bit hard to read). --- ordered-trie/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index e30ffac1..33236135 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -917,7 +917,8 @@ impl MultiProofState { } } - let right = self.next_key1.as_ref() + // sibling || is just to skip next nible query as it is unused. + let right = sibling || self.next_key1.as_ref() .expect("start_with_current").nibble_at(key.depth()).expect("starts with"); if let Some(join1) = self.join1 { if let Some(join2) = self.join2 { @@ -939,13 +940,13 @@ impl MultiProofState { unreachable!("next is defined (start_with_next)"); } if !sibling { - // TODO could skip right resolution in sibling case if right { return MultiProofResult::RegisterLeft; } else { return MultiProofResult::RegisterRight; } } + return MultiProofResult::DoNothing; } MultiProofResult::DoNothingNoStartWith } From 1a4295da034220b27982740f3dfa8d388e6ee28d Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 17:18:24 +0200 Subject: [PATCH 15/73] some todos fixed. --- test-support/reference-trie/src/lib.rs | 5 +---- trie-db/src/lib.rs | 1 - trie-db/src/trie_codec.rs | 9 +-------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index dc9fac6f..abe0de95 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -963,11 +963,8 @@ fn encode_proof_internal( let bitmap_start = result.len(); result.push(0u8); result.push(0u8); - // write all inline nodes TODO we could omit children first - // as in std case and fill this bitmap as in generate.rs. + // Write all inline nodes. for (ix, child) in children.iter().enumerate() { - // TODO EMCH seems like we do not need in_proof_children input - // How does it differs from standard bitmap?? if let Some(ChildReference::Inline(h, nb)) = child.borrow() { if *nb > 0 { // TODO do not write inline of null size, these are defined diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 88c59aa8..b17f0cfa 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -388,7 +388,6 @@ pub trait TrieLayout { /// Note that if does not, the `NodeCodecComplex` hash /// associated codec only really need to implement `NodeCodec` /// and dummy implementation can be used. - /// TODO EMCH consider splitting trait with a TrieLayoutComplex variant!! const COMPLEX_HASH: bool; /// Hasher to use for this trie. type Hash: BinaryHasher; diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 360e5697..7ad8451b 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -578,13 +578,6 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) /// indicates if it is included in the proof (inline node or /// compacted node). /// - `hash_buf` a buffer of the right size to compute the hash. -/// -/// TODO EMCH this can be highly optimized: we do not need to calculate -/// hash up to the root but only up to the needed intermediate hash. -/// -> custom callback different than trie_root probably -/// at this time trie_root run on empty inline node and produce some incorrect -/// path that cannot be included in additional hash. -/// TODO and can move to ordered_trie?? pub fn binary_additional_hashes( children: &[Option>], in_proof_children: &[bool], @@ -611,7 +604,7 @@ pub fn binary_additional_hashes( ChildReference::Hash(h) => h.clone(), ChildReference::Inline(h, _) => h.clone(), }); - // TODO we can skip a hash (the one to calculate root), actually we can skip all hash + // Note that since call back skip some calculation, root result cannot be use in this case. let _root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.clone().into_iter(), &mut callback); callback.take_additional_hash() } From 80efb76657420775ce301bf8a4a216b9485811a1 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 17:32:00 +0200 Subject: [PATCH 16/73] some cleaning and renaming. --- hash-db/src/lib.rs | 61 +------------------------- memory-db/src/lib.rs | 7 ++- test-support/reference-trie/src/lib.rs | 22 +++++----- trie-db/src/iter_build.rs | 36 +++++++-------- trie-db/src/lib.rs | 2 +- trie-db/src/node_codec.rs | 51 ++++++--------------- trie-db/src/proof/verify.rs | 12 ++--- trie-db/src/trie_codec.rs | 16 +++---- trie-db/src/triedbmut.rs | 14 +++--- 9 files changed, 69 insertions(+), 152 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 8272d23e..249ae4eb 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -32,6 +32,7 @@ pub trait MaybeDebug {} #[cfg(not(feature = "std"))] impl MaybeDebug for T {} + /// A trie node prefix, it is the nibble path from the trie root /// to the trie node. /// For a node containing no partial key value it is the full key. @@ -129,18 +130,6 @@ pub trait HashDB: Send + Sync + AsHashDB { fn remove(&mut self, key: &H::Out, prefix: Prefix); } -/// Describes how to hash a node given its layout -pub struct ComplexLayout { - pub nb_children: usize, - // can be calculated from decoded node from the - // children bitmap and the children range - pub children: I, // impl iterator < (is_defined, range) > - pub nb_additional_hashes: usize, - // TODO switch to iter?? - pub additional_hashes: I2, -} - - /// Trait for immutable reference of HashDB. pub trait HashDBRef { /// Look up a given hash into the bytes that hash to it, returning None if the @@ -196,51 +185,3 @@ impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { fn as_plain_db(&self) -> &dyn PlainDB { &**self } fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn PlainDB + 'b) { &mut **self } } - -/* - -TODO this is rather shit at this point not delete yet just in case - -/// Fix hash implementation, it needs to have -/// same output length as input for rounds of hashing. -/// TODO consider moving in its own crate. -pub trait FixHash { - type Hasher: Hasher; - /// if true, then when processing two leaf we do a finalize round. - /// TODO I think only one may be insecure but do not remember - /// the rational, also this should be an associated constant - /// (TODO group types parameters) - /// - /// Tells if state/iv can be initiated from first element of pair. - /// - /// TODO could it be skipped in any circumstance or the other way - /// arount, todo check blake3 permutation - /// - /// Need first hashed implies that we cannot use back a state by - /// calling a second hash. - /// TODO write a test case for that !!! - /// TODO rename to need finalize?? - /// TODO for keccack we could only hash(hash1 xor hash2)? - const NEED_FIRST_HASHED: bool; - /// Value of empty hash at some given depth - /// TODO test case it - const EMPTY_HASHES: &'static [&'static [u8]]; - - // The fix hash type of the `Hasher`. -> is Hasher::Out -// type Out: AsRef<[u8]> + AsMut<[u8]> + Default + hash_db::MaybeDebug + PartialEq + Eq -// + Send + Sync + Clone + Copy; - - // The length in bytes of the `Out` output. -> is Hasher::Length -// const LENGTH: usize; - - /// Compute the hash - fn new(first: ::Out) -> Self; - /// Compute the hash - fn hash(&mut self, second: &::Out); - /// Access current state (if NEED_FIRST_HASHED is false). - fn current_state(&self) -> &::Out; - /// Extract hash (if NEED_FIRST_HASHED is true). - fn finalize(self) -> ::Out; -} -*/ - diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index c8db4c9f..eb280f88 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -615,7 +615,7 @@ where &mut self, prefix: Prefix, value: &[u8], - no_child_value: &[u8], + child_proof_header: &[u8], nb_children: usize, children: I, additional_hashes: I2, @@ -626,7 +626,7 @@ where } let key = if let Some(key) = H::hash_complex( - no_child_value, + child_proof_header, nb_children, children, additional_hashes, @@ -634,8 +634,7 @@ where ) { key } else { - // invalid proof TODO do dedicated method that can return - // error?? + // invalid proof TODO handle error properly return self.hashed_null_node.clone(); }; diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index abe0de95..c1f1e239 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -30,7 +30,7 @@ use trie_db::{ TrieRootComplex, Partial, BinaryHasher, - EncodedCommon, + ChildRootHeader, HashesPlan, binary_additional_hashes, }; @@ -792,7 +792,7 @@ impl NodeCodecComplex for ReferenceNodeCodec { children: impl Iterator>>>, maybe_value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedCommon) { + ) -> (Vec, ChildRootHeader) { Self::branch_node_internal(children, maybe_value, Some(register_children), true) } @@ -802,7 +802,7 @@ impl NodeCodecComplex for ReferenceNodeCodec { _children: impl Iterator>>>, _maybe_value: Option<&[u8]>, _register_children: &mut [Option>], - ) -> (Vec, EncodedCommon) { + ) -> (Vec, ChildRootHeader) { unreachable!() } @@ -838,7 +838,7 @@ impl ReferenceNodeCodec { maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, encode_children: bool, - ) -> (Vec, EncodedCommon) { + ) -> (Vec, ChildRootHeader) { let mut output = vec![0; BITMAP_LENGTH + 1]; let mut prefix: [u8; 3] = [0; 3]; let have_value = if let Some(value) = maybe_value { @@ -852,12 +852,12 @@ impl ReferenceNodeCodec { let mut register_children = register_children.as_mut(); let register_children = &mut register_children; let common = if encode_children && register_children.is_some() { - EncodedCommon::Range(Range { + ChildRootHeader::Range(Range { start: 0, end: output.len(), }) } else { - EncodedCommon::Unused + ChildRootHeader::Unused }; let mut child_ix = output.len(); @@ -1126,7 +1126,7 @@ impl ReferenceNodeCodecNoExt { maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, encode_children: bool, - ) -> (Vec, EncodedCommon) { + ) -> (Vec, ChildRootHeader) { let mut output = if maybe_value.is_some() { partial_from_iterator_encode( partial, @@ -1151,12 +1151,12 @@ impl ReferenceNodeCodecNoExt { let mut register_children = register_children.as_mut(); let register_children = &mut register_children; let common = if encode_children && register_children.is_some() { - EncodedCommon::Range(Range { + ChildRootHeader::Range(Range { start: 0, end: output.len(), }) } else { - EncodedCommon::Unused + ChildRootHeader::Unused }; let mut child_ix = output.len(); @@ -1218,7 +1218,7 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { _children: impl Iterator::Out>>>>, _maybe_value: Option<&[u8]>, _register_children: &mut [Option>], - ) -> (Vec, EncodedCommon) { + ) -> (Vec, ChildRootHeader) { unreachable!() } @@ -1228,7 +1228,7 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { children: impl Iterator>>>, maybe_value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedCommon) { + ) -> (Vec, ChildRootHeader) { Self::branch_node_nibbled_internal( partial, number_nibble, diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 46c4e073..70ea8739 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -23,7 +23,7 @@ use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; use crate::nibble::nibble_ops; -use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedCommon}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildRootHeader}; use crate::{TrieLayout, TrieHash}; use crate::rstd::borrow::Borrow; @@ -146,7 +146,7 @@ impl CacheAccum k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); let iter: Option<(EmptyIter>, _)> = None; - let hash = callback.process(pr.left(), (encoded, EncodedCommon::Unused), false, iter); + let hash = callback.process(pr.left(), (encoded, ChildRootHeader::Unused), false, iter); // insert hash in branch (first level branch only at this point) self.set_node(target_depth, nibble_value as usize, Some(hash)); @@ -219,7 +219,7 @@ impl CacheAccum (T::Codec::branch_node( self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) }; let pr = NibbleSlice::new_offset(&key_branch, branch_d); let branch_hash = if T::COMPLEX_HASH { @@ -237,7 +237,7 @@ impl CacheAccum let nib = pr.right_range_iter(nkeyix.1); let encoded = T::Codec::extension_node(nib, nkeyix.1, branch_hash); let iter: Option<(EmptyIter>, _)> = None; - let h = callback.process(pr.left(), (encoded, EncodedCommon::Unused), is_root, iter); + let h = callback.process(pr.left(), (encoded, ChildRootHeader::Unused), is_root, iter); h } else { branch_hash @@ -272,7 +272,7 @@ impl CacheAccum pr.right_range_iter(nkeyix.1), nkeyix.1, self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) }; let ext_length = nkey.as_ref().map(|nkeyix| nkeyix.0).unwrap_or(0); let pr = NibbleSlice::new_offset( @@ -347,7 +347,7 @@ pub fn trie_visit(input: I, callback: &mut F) k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); let iter: Option<(EmptyIter>, _)> = None; - callback.process(pr.left(), (encoded, EncodedCommon::Unused), true, iter); + callback.process(pr.left(), (encoded, ChildRootHeader::Unused), true, iter); } else { depth_queue.flush_value(callback, last_depth, &previous_value); let ref_branches = previous_value.0; @@ -356,7 +356,7 @@ pub fn trie_visit(input: I, callback: &mut F) } else { let iter: Option<(EmptyIter>, _)> = None; // nothing null root corner case - callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), EncodedCommon::Unused), true, iter); + callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), ChildRootHeader::Unused), true, iter); } } @@ -372,7 +372,7 @@ pub trait ProcessEncodedNode { fn process( &mut self, prefix: Prefix, - encoded_node: (Vec, EncodedCommon), + encoded_node: (Vec, ChildRootHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference; @@ -414,7 +414,7 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> fn process( &mut self, prefix: Prefix, - (encoded_node, _common): (Vec, EncodedCommon), + (encoded_node, _common): (Vec, ChildRootHeader), is_root: bool, // TODO different trait?? _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -439,7 +439,7 @@ impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode<, EncodedCommon), + (encoded_node, common): (Vec, ChildRootHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -461,7 +461,7 @@ impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode< ProcessEncodedNode<::Out> for TrieRoot, EncodedCommon), + (encoded_node, _common): (Vec, ChildRootHeader), is_root: bool, // TODO different trait _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -531,7 +531,7 @@ impl ProcessEncodedNode<::Out> for TrieRootComple fn process( &mut self, _: Prefix, - (encoded_node, common): (Vec, EncodedCommon), + (encoded_node, common): (Vec, ChildRootHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -550,7 +550,7 @@ impl ProcessEncodedNode<::Out> for TrieRootComple None => None, }); ::hash_complex( - common.encoded_common(&encoded_node[..]), + common.header(&encoded_node[..]), nb_children, iter, EmptyIter::default(), @@ -615,7 +615,7 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, EncodedCommon), + (encoded_node, _common): (Vec, ChildRootHeader), is_root: bool, // TODO different trait? _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -643,7 +643,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { fn process( &mut self, _: Prefix, - (encoded_node, _common): (Vec, EncodedCommon), + (encoded_node, _common): (Vec, ChildRootHeader), is_root: bool, // TODO different trait _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -667,7 +667,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhash fn process( &mut self, _: Prefix, - (encoded_node, common): (Vec, EncodedCommon), + (encoded_node, common): (Vec, ChildRootHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -686,7 +686,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhash None => None, }); ::hash_complex( - common.encoded_common(&encoded_node[..]), + common.header(&encoded_node[..]), nb_children, iter, EmptyIter::default(), diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index b17f0cfa..20ab4e0e 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -70,7 +70,7 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, EncodedCommon, +pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, ChildRootHeader, Bitmap, BITMAP_LENGTH, HashesPlan}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 5d73447a..8199e954 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -128,7 +128,7 @@ pub trait NodeCodecComplex: NodeCodec { children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedCommon); + ) -> (Vec, ChildRootHeader); /// Variant of `branch_node_common` but with a nibble. /// @@ -140,10 +140,10 @@ pub trait NodeCodecComplex: NodeCodec { children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, EncodedCommon); + ) -> (Vec, ChildRootHeader); /// Returns branch node encoded information for hash. - /// Result is the same as `branch_node_common().1.encoded_common(branch_node_common().0`. + /// Result is the same as `branch_node_common().1.header(branch_node_common().0`. fn branch_node_for_hash( children: impl Iterator>>>, value: Option<&[u8]>, @@ -161,16 +161,13 @@ pub trait NodeCodecComplex: NodeCodec { /// /// - `hash_proof_header`: the part common with the header info from hash. /// It can be calculated from `branch_node_common` through - /// `EncodedCommon` call, or directly by `branch_node_for_hash`. - /// TODO EMCH rename this common `HashProofHeader`. + /// `ChildRootHeader` call, or directly by `branch_node_for_hash`. /// - `children`: contains all children, with compact (ommited children) defined as /// a null length inline node. /// The children to be include in the proof are therefore the compacted one and the /// inline nodes only. /// The other children value are needed because they can be included into the additional /// hash, and are required for intermediate hash calculation. - /// - /// TODO consider using bitmap directly fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], @@ -180,10 +177,9 @@ pub trait NodeCodecComplex: NodeCodec { /// Information to fetch bytes that needs to be include when calculating a node hash. /// The node hash is the hash of these information and the merkle root of its children. -/// TODO EMCH rename to BranchHashInfo #[derive(Clone)] -pub enum EncodedCommon { - /// No need for complex hash. TODO EMCH see if still used. +pub enum ChildRootHeader { + /// No need for complex hash. Unused, /// Range over the branch encoded for storage. Range(Range), @@ -191,33 +187,14 @@ pub enum EncodedCommon { Allocated(Vec), } -impl EncodedCommon { - pub fn encoded_common<'a>(&'a self, encoded: &'a [u8]) -> &'a [u8] { +impl ChildRootHeader { + pub fn header<'a>(&'a self, encoded: &'a [u8]) -> &'a [u8] { match self { - EncodedCommon::Unused => encoded, - EncodedCommon::Range(range) => &encoded[range.clone()], - EncodedCommon::Allocated(buff) => &buff[..], + ChildRootHeader::Unused => encoded, + ChildRootHeader::Range(range) => &encoded[range.clone()], + ChildRootHeader::Allocated(buff) => &buff[..], } } - // TODO this is bad we should produce a branch that does - // not include it in the first place (new encode fn with - // default impl using trim no child). - // TODO consider removal - pub fn trim_common(self, encoded: &mut Vec) { - match self { - EncodedCommon::Unused => (), - EncodedCommon::Range(range) => { - encoded.truncate(range.end); - if range.start != 0 { - *encoded = encoded.split_off(range.start); - } - }, - EncodedCommon::Allocated(buf) => { - *encoded = buf; - }, - } - } - } use ordered_trie::{HashDBComplex, HasherComplex}; @@ -235,7 +212,7 @@ pub trait HashDBComplexDyn: Send + Sync + HashDB { prefix: Prefix, value: &[u8], children: &[Option>], - common: EncodedCommon, + common: ChildRootHeader, ) -> H::Out; } @@ -245,7 +222,7 @@ impl> HashDBComplexDyn for C { prefix: Prefix, value: &[u8], children: &[Option>], - common: EncodedCommon, + common: ChildRootHeader, ) -> H::Out { // TODO factor this with iter_build (just use the trait) @@ -259,7 +236,7 @@ impl> HashDBComplexDyn for C { self, prefix, value, - common.encoded_common(value), + common.header(value), nb_children, children, EmptyIter::default(), diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index d9be95e7..ddba1f35 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -17,7 +17,7 @@ use crate::rstd::{ }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, EncodedCommon, + node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, ChildRootHeader, }; use hash_db::Hasher; use ordered_trie::{BinaryHasher, HasherComplex}; @@ -149,11 +149,11 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> } /// Encode this entry to an encoded trie node with data properly reconstructed. - fn encode_node(&mut self) -> Result<(Vec, EncodedCommon), Error> { + fn encode_node(&mut self) -> Result<(Vec, ChildRootHeader), Error> { self.complete_children()?; Ok(match self.node { Node::Empty => - (C::empty_node().to_vec(), EncodedCommon::Unused), + (C::empty_node().to_vec(), ChildRootHeader::Unused), Node::Leaf(partial, _) => { let value = self.value .expect( @@ -161,7 +161,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> value is only ever reassigned in the ValueMatch::MatchesLeaf match \ clause, which assigns only to Some" ); - (C::leaf_node(partial.right(), value), EncodedCommon::Unused) + (C::leaf_node(partial.right(), value), ChildRootHeader::Unused) } Node::Extension(partial, _) => { let child = self.children[0] @@ -170,7 +170,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> partial.right_iter(), partial.len(), child - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) } Node::Branch(_, _) => { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); @@ -504,7 +504,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve }); if let Some(h) = L::Hash::hash_complex( - &common.encoded_common(node_data.as_slice())[..], + &common.header(node_data.as_slice())[..], nb_children, children, additional_hash.into_iter(), diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 7ad8451b..4133446e 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -31,7 +31,7 @@ use crate::{ TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecComplex, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; -use crate::node_codec::{Bitmap, EncodedCommon}; +use crate::node_codec::{Bitmap, ChildRootHeader}; use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, rc::Rc, result, vec, vec::Vec, ops::Range, @@ -414,19 +414,19 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { /// /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. - fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, EncodedCommon) { + fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, ChildRootHeader) { match self.node { Node::Empty => - (C::empty_node().to_vec(), EncodedCommon::Unused), + (C::empty_node().to_vec(), ChildRootHeader::Unused), Node::Leaf(partial, value) => - (C::leaf_node(partial.right(), value), EncodedCommon::Unused), + (C::leaf_node(partial.right(), value), ChildRootHeader::Unused), Node::Extension(partial, _) => (C::extension_node( partial.right_iter(), partial.len(), self.children[0] .expect("required by method precondition; qed"), - ), EncodedCommon::Unused), + ), ChildRootHeader::Unused), Node::Branch(_, value) => if let Some(register_children) = register_children { C::branch_node_common( self.children.into_iter(), @@ -437,7 +437,7 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { (C::branch_node( self.children.into_iter(), value, - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) }, Node::NibbledBranch(partial, _, value) => if let Some(register_children) = register_children { C::branch_node_nibbled_common( @@ -453,7 +453,7 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { partial.len(), self.children.iter(), value, - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) }, } } @@ -547,7 +547,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) db.insert_complex( prefix.as_prefix(), &node_data[..], - common.encoded_common(&node_data[..]), + common.header(&node_data[..]), nb_children, children, additional_hashes, diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 57411829..a9cb638a 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -23,7 +23,7 @@ use crate::node_codec::HashDBComplexDyn; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::node_codec::{NodeCodec, NodeCodecComplex, EncodedCommon}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildRootHeader}; use crate::nibble::{NibbleVec, NibbleSlice, nibble_ops, BackingByteVec}; use crate::rstd::{ boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, @@ -209,16 +209,16 @@ where self, mut child_cb: F, register_children: Option<&mut [Option>]>, - ) -> (Vec, EncodedCommon) where + ) -> (Vec, ChildRootHeader) where C: NodeCodecComplex, F: FnMut(NodeHandle, Option<&NibbleSlice>, Option) -> ChildReference, H: Hasher, { match self { - Node::Empty => (C::empty_node().to_vec(), EncodedCommon::Unused), + Node::Empty => (C::empty_node().to_vec(), ChildRootHeader::Unused), Node::Leaf(partial, value) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - (C::leaf_node(pr.right(), &value), EncodedCommon::Unused) + (C::leaf_node(pr.right(), &value), ChildRootHeader::Unused) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); @@ -228,7 +228,7 @@ where it, pr.len(), c, - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) }, Node::Branch(mut children, value) => { // map the `NodeHandle`s from the Branch to `ChildReferences` @@ -249,7 +249,7 @@ where (C::branch_node( children, value.as_ref().map(|v| &v[..]), - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) } }, Node::NibbledBranch(partial, mut children, value) => { @@ -281,7 +281,7 @@ where pr.len(), children, value.as_ref().map(|v| &v[..]), - ), EncodedCommon::Unused) + ), ChildRootHeader::Unused) } }, } From b01b2dcec3d11ec09960dee4412c4857d4bfe98b Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 20:57:00 +0200 Subject: [PATCH 17/73] A quite involve fix (KeyNode trait needs additional documentation). --- ordered-trie/src/lib.rs | 55 ++++++++++++++--------------------------- trie-db/fuzz/src/lib.rs | 7 ++++++ 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 33236135..df61c9e9 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -344,6 +344,7 @@ impl SequenceBinaryTree { } }) } + pub fn iter_path_node_key(&self, from: Option) -> impl Iterator where KN: KeyNode + From<(usize, usize)> + Clone, @@ -453,7 +454,6 @@ impl SequenceBinaryTree { // then the value just need to be extract from terminal hash instead. (terminal hash marker // and value describe above is still interesting). - /// key of node is a sequence of one bit nibbles. pub trait KeyNode { fn depth(&self) -> usize; @@ -475,7 +475,7 @@ pub trait KeyNode { // please do not use, only for test of (usize, K) struct VecKeyNode(std::collections::VecDeque); #[cfg(test)] -impl KeyNode for VecKeyNode { +impl KeyNode for VecKeyNode { fn increment_no_increase(&mut self) { for i in (0..self.0.len()).rev() { match self.0.get_mut(i) { @@ -494,6 +494,7 @@ impl KeyNode for VecKeyNode { fn depth(&self) -> usize { self.0.len() } + fn nibble_at(&self, depth: usize) -> Option { self.0.get(depth).cloned() } @@ -538,34 +539,6 @@ impl From<(usize, usize)> for VecKeyNode { if depth == 0 { return VecKeyNode(std::collections::VecDeque::new()); } -/* if depth == 0 { - return vec![]; - return vec![0]; - return vec![0, 0]; - ... - } - if depth == 1 { - return vec![1]; - return vec![0, 1]; - return vec![0, 0, 1]; - } - if depth == 2 { - return vec![1, 0]; - return vec![0, 1, 0]; - } - if depth == 3 { - return vec![1, 1]; - return vec![0, 1, 1]; - } - if depth == 4 { - return vec![1, 0, 0]; - } - if depth == 5 { - return vec![1, 0, 1]; - } -*/ - - VecKeyNode( (1..=depth).map(|i| right_at(key, depth - i)).collect() ) @@ -664,6 +637,16 @@ impl KeyNode for UsizeKeyNode { } } fn common_depth(&self, other: &Self) -> usize { +/* let bound = crate::rstd::cmp::min(self.depth, other.depth); + let mut depth = 0; + for i in 0..bound { + if self.nibble_at(i) == other.nibble_at(i) { + depth += 1; + } else { + break; + } + } + depth*/ let (big, small) = if self.depth < other.depth { (other, self) } else { @@ -672,7 +655,6 @@ impl KeyNode for UsizeKeyNode { // end is not common let big_v = big.value >> (big.depth - small.depth); let diff = big_v ^ small.value; - small.depth - (0usize.leading_zeros() - diff.leading_zeros()) as usize } } @@ -1528,7 +1510,7 @@ impl HasherComplex for H { I: Iterator::Out>>, I2: Iterator::Out>, >( - x: &[u8], + header: &[u8], nb_children: usize, children: I, additional_hashes: I2, @@ -1543,8 +1525,9 @@ impl HasherComplex for H { let iter = children.filter_map(|v| v); // TODO assert same number as count crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof) } else { - // proof node - let iter_key = seq_trie.iter_depth(None).enumerate().map(Into::::into); + // proof node, UsizeKeyNode should be big enough for hasher complex + // case. + let iter_key = seq_trie.iter_path_node_key::(None); let iter = children .zip(iter_key) .filter_map(|(value, key)| if let Some(value) = value { @@ -1565,8 +1548,8 @@ impl HasherComplex for H { } }; // TODO really need a real hash trait - let mut buf = Vec::with_capacity(x.len() + hash.as_ref().len()); - buf.extend_from_slice(x); + let mut buf = Vec::with_capacity(header.len() + hash.as_ref().len()); + buf.extend_from_slice(header); buf.extend_from_slice(hash.as_ref()); Some(H::hash(buf.as_slice())) } diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index c329ccfd..3f36debf 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -363,3 +363,10 @@ fn test_generate_proof( (root, proof, items) } + +/* +#[test] +fn debug_error() { + let input = [0x40,0x0,0xe6,0xff,0xff,0x40,0x0,0xe6,0xff,0xff,0x2b]; + fuzz_that_verify_accepts_valid_proofs(&input[..]); +}*/ From 7a0ff5becafb39473cf49551dbe8a3fbe4128ecb Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 20:59:23 +0200 Subject: [PATCH 18/73] quick mesg --- ordered-trie/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index df61c9e9..8bb0797b 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -455,6 +455,10 @@ impl SequenceBinaryTree { // and value describe above is still interesting). /// key of node is a sequence of one bit nibbles. +/// This do not implement any key alignment logic, +/// using it for the sequence trie should always +/// use `iter_path_node_key` or `path_node_key` +/// function to instantiate. pub trait KeyNode { fn depth(&self) -> usize; // return nibble at depth (bin tree so return bool) From bad03f5b61f60e4e8aec825e0a1859e9924657ae Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 21:20:10 +0200 Subject: [PATCH 19/73] add rstd vec --- ordered-trie/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 8bb0797b..a836cfe1 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -64,6 +64,7 @@ pub mod key { pub mod meta { use hash_db::Hasher; + use self::rstd::vec::Vec; /// Codec for the meta needed to get /// information for the tree. From 24fb5a4c98acee125b7ad620c40ffe0c5b5bd594 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Apr 2020 21:21:07 +0200 Subject: [PATCH 20/73] fix --- ordered-trie/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index a836cfe1..cb924a78 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -40,7 +40,7 @@ mod rstd { use hash_db::{Prefix, HashDB}; -use self::rstd::vec::Vec; +use crate::rstd::vec::Vec; use hash_db::Hasher; use crate::rstd::marker::PhantomData; @@ -64,7 +64,7 @@ pub mod key { pub mod meta { use hash_db::Hasher; - use self::rstd::vec::Vec; + use crate::rstd::vec::Vec; /// Codec for the meta needed to get /// information for the tree. From e3595786099bc2a67d11b40321feca58238995df Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 11:37:13 +0200 Subject: [PATCH 21/73] rename ChildRootHeader to ChildProofHeader --- test-support/reference-trie/src/lib.rs | 23 ++++++++++---------- trie-db/src/iter_build.rs | 30 +++++++++++++------------- trie-db/src/lib.rs | 2 +- trie-db/src/node_codec.rs | 20 ++++++++--------- trie-db/src/proof/verify.rs | 10 ++++----- trie-db/src/trie_codec.rs | 14 ++++++------ trie-db/src/triedbmut.rs | 14 ++++++------ 7 files changed, 56 insertions(+), 57 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index c1f1e239..5ed0f1d6 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -30,7 +30,7 @@ use trie_db::{ TrieRootComplex, Partial, BinaryHasher, - ChildRootHeader, + ChildProofHeader, HashesPlan, binary_additional_hashes, }; @@ -792,7 +792,7 @@ impl NodeCodecComplex for ReferenceNodeCodec { children: impl Iterator>>>, maybe_value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, ChildRootHeader) { + ) -> (Vec, ChildProofHeader) { Self::branch_node_internal(children, maybe_value, Some(register_children), true) } @@ -802,7 +802,7 @@ impl NodeCodecComplex for ReferenceNodeCodec { _children: impl Iterator>>>, _maybe_value: Option<&[u8]>, _register_children: &mut [Option>], - ) -> (Vec, ChildRootHeader) { + ) -> (Vec, ChildProofHeader) { unreachable!() } @@ -829,7 +829,6 @@ impl NodeCodecComplex for ReferenceNodeCodec { ) -> Vec { encode_proof_internal::(hash_proof_header, children, hash_buf) } - } impl ReferenceNodeCodec { @@ -838,7 +837,7 @@ impl ReferenceNodeCodec { maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, encode_children: bool, - ) -> (Vec, ChildRootHeader) { + ) -> (Vec, ChildProofHeader) { let mut output = vec![0; BITMAP_LENGTH + 1]; let mut prefix: [u8; 3] = [0; 3]; let have_value = if let Some(value) = maybe_value { @@ -852,12 +851,12 @@ impl ReferenceNodeCodec { let mut register_children = register_children.as_mut(); let register_children = &mut register_children; let common = if encode_children && register_children.is_some() { - ChildRootHeader::Range(Range { + ChildProofHeader::Range(Range { start: 0, end: output.len(), }) } else { - ChildRootHeader::Unused + ChildProofHeader::Unused }; let mut child_ix = output.len(); @@ -1126,7 +1125,7 @@ impl ReferenceNodeCodecNoExt { maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, encode_children: bool, - ) -> (Vec, ChildRootHeader) { + ) -> (Vec, ChildProofHeader) { let mut output = if maybe_value.is_some() { partial_from_iterator_encode( partial, @@ -1151,12 +1150,12 @@ impl ReferenceNodeCodecNoExt { let mut register_children = register_children.as_mut(); let register_children = &mut register_children; let common = if encode_children && register_children.is_some() { - ChildRootHeader::Range(Range { + ChildProofHeader::Range(Range { start: 0, end: output.len(), }) } else { - ChildRootHeader::Unused + ChildProofHeader::Unused }; let mut child_ix = output.len(); @@ -1218,7 +1217,7 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { _children: impl Iterator::Out>>>>, _maybe_value: Option<&[u8]>, _register_children: &mut [Option>], - ) -> (Vec, ChildRootHeader) { + ) -> (Vec, ChildProofHeader) { unreachable!() } @@ -1228,7 +1227,7 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { children: impl Iterator>>>, maybe_value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, ChildRootHeader) { + ) -> (Vec, ChildProofHeader) { Self::branch_node_nibbled_internal( partial, number_nibble, diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 70ea8739..d8e128e8 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -23,7 +23,7 @@ use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; use crate::nibble::nibble_ops; -use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildRootHeader}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildProofHeader}; use crate::{TrieLayout, TrieHash}; use crate::rstd::borrow::Borrow; @@ -146,7 +146,7 @@ impl CacheAccum k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); let iter: Option<(EmptyIter>, _)> = None; - let hash = callback.process(pr.left(), (encoded, ChildRootHeader::Unused), false, iter); + let hash = callback.process(pr.left(), (encoded, ChildProofHeader::Unused), false, iter); // insert hash in branch (first level branch only at this point) self.set_node(target_depth, nibble_value as usize, Some(hash)); @@ -219,7 +219,7 @@ impl CacheAccum (T::Codec::branch_node( self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) }; let pr = NibbleSlice::new_offset(&key_branch, branch_d); let branch_hash = if T::COMPLEX_HASH { @@ -237,7 +237,7 @@ impl CacheAccum let nib = pr.right_range_iter(nkeyix.1); let encoded = T::Codec::extension_node(nib, nkeyix.1, branch_hash); let iter: Option<(EmptyIter>, _)> = None; - let h = callback.process(pr.left(), (encoded, ChildRootHeader::Unused), is_root, iter); + let h = callback.process(pr.left(), (encoded, ChildProofHeader::Unused), is_root, iter); h } else { branch_hash @@ -272,7 +272,7 @@ impl CacheAccum pr.right_range_iter(nkeyix.1), nkeyix.1, self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) }; let ext_length = nkey.as_ref().map(|nkeyix| nkeyix.0).unwrap_or(0); let pr = NibbleSlice::new_offset( @@ -347,7 +347,7 @@ pub fn trie_visit(input: I, callback: &mut F) k2.as_ref().len() * nibble_ops::NIBBLE_PER_BYTE - nkey.len(), ); let iter: Option<(EmptyIter>, _)> = None; - callback.process(pr.left(), (encoded, ChildRootHeader::Unused), true, iter); + callback.process(pr.left(), (encoded, ChildProofHeader::Unused), true, iter); } else { depth_queue.flush_value(callback, last_depth, &previous_value); let ref_branches = previous_value.0; @@ -356,7 +356,7 @@ pub fn trie_visit(input: I, callback: &mut F) } else { let iter: Option<(EmptyIter>, _)> = None; // nothing null root corner case - callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), ChildRootHeader::Unused), true, iter); + callback.process(hash_db::EMPTY_PREFIX, (T::Codec::empty_node().to_vec(), ChildProofHeader::Unused), true, iter); } } @@ -372,7 +372,7 @@ pub trait ProcessEncodedNode { fn process( &mut self, prefix: Prefix, - encoded_node: (Vec, ChildRootHeader), + encoded_node: (Vec, ChildProofHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference; @@ -414,7 +414,7 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> fn process( &mut self, prefix: Prefix, - (encoded_node, _common): (Vec, ChildRootHeader), + (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, // TODO different trait?? _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -439,7 +439,7 @@ impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode<, ChildRootHeader), + (encoded_node, common): (Vec, ChildProofHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -494,7 +494,7 @@ impl ProcessEncodedNode<::Out> for TrieRoot, ChildRootHeader), + (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, // TODO different trait _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -531,7 +531,7 @@ impl ProcessEncodedNode<::Out> for TrieRootComple fn process( &mut self, _: Prefix, - (encoded_node, common): (Vec, ChildRootHeader), + (encoded_node, common): (Vec, ChildProofHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { @@ -615,7 +615,7 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, ChildRootHeader), + (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, // TODO different trait? _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -643,7 +643,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { fn process( &mut self, _: Prefix, - (encoded_node, _common): (Vec, ChildRootHeader), + (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, // TODO different trait _complex_hash: Option<(impl Iterator>>>, usize)>, @@ -667,7 +667,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhash fn process( &mut self, _: Prefix, - (encoded_node, common): (Vec, ChildRootHeader), + (encoded_node, common): (Vec, ChildProofHeader), is_root: bool, complex_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 20ab4e0e..2dd73fa8 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -70,7 +70,7 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, ChildRootHeader, +pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, ChildProofHeader, Bitmap, BITMAP_LENGTH, HashesPlan}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 8199e954..f784190f 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -128,7 +128,7 @@ pub trait NodeCodecComplex: NodeCodec { children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, ChildRootHeader); + ) -> (Vec, ChildProofHeader); /// Variant of `branch_node_common` but with a nibble. /// @@ -140,7 +140,7 @@ pub trait NodeCodecComplex: NodeCodec { children: impl Iterator>>>, value: Option<&[u8]>, register_children: &mut [Option>], - ) -> (Vec, ChildRootHeader); + ) -> (Vec, ChildProofHeader); /// Returns branch node encoded information for hash. /// Result is the same as `branch_node_common().1.header(branch_node_common().0`. @@ -161,7 +161,7 @@ pub trait NodeCodecComplex: NodeCodec { /// /// - `hash_proof_header`: the part common with the header info from hash. /// It can be calculated from `branch_node_common` through - /// `ChildRootHeader` call, or directly by `branch_node_for_hash`. + /// `ChildProofHeader` call, or directly by `branch_node_for_hash`. /// - `children`: contains all children, with compact (ommited children) defined as /// a null length inline node. /// The children to be include in the proof are therefore the compacted one and the @@ -178,7 +178,7 @@ pub trait NodeCodecComplex: NodeCodec { /// Information to fetch bytes that needs to be include when calculating a node hash. /// The node hash is the hash of these information and the merkle root of its children. #[derive(Clone)] -pub enum ChildRootHeader { +pub enum ChildProofHeader { /// No need for complex hash. Unused, /// Range over the branch encoded for storage. @@ -187,12 +187,12 @@ pub enum ChildRootHeader { Allocated(Vec), } -impl ChildRootHeader { +impl ChildProofHeader { pub fn header<'a>(&'a self, encoded: &'a [u8]) -> &'a [u8] { match self { - ChildRootHeader::Unused => encoded, - ChildRootHeader::Range(range) => &encoded[range.clone()], - ChildRootHeader::Allocated(buff) => &buff[..], + ChildProofHeader::Unused => encoded, + ChildProofHeader::Range(range) => &encoded[range.clone()], + ChildProofHeader::Allocated(buff) => &buff[..], } } } @@ -212,7 +212,7 @@ pub trait HashDBComplexDyn: Send + Sync + HashDB { prefix: Prefix, value: &[u8], children: &[Option>], - common: ChildRootHeader, + common: ChildProofHeader, ) -> H::Out; } @@ -222,7 +222,7 @@ impl> HashDBComplexDyn for C { prefix: Prefix, value: &[u8], children: &[Option>], - common: ChildRootHeader, + common: ChildProofHeader, ) -> H::Out { // TODO factor this with iter_build (just use the trait) diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index ddba1f35..a8f84dca 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -17,7 +17,7 @@ use crate::rstd::{ }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, ChildRootHeader, + node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, ChildProofHeader, }; use hash_db::Hasher; use ordered_trie::{BinaryHasher, HasherComplex}; @@ -149,11 +149,11 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> } /// Encode this entry to an encoded trie node with data properly reconstructed. - fn encode_node(&mut self) -> Result<(Vec, ChildRootHeader), Error> { + fn encode_node(&mut self) -> Result<(Vec, ChildProofHeader), Error> { self.complete_children()?; Ok(match self.node { Node::Empty => - (C::empty_node().to_vec(), ChildRootHeader::Unused), + (C::empty_node().to_vec(), ChildProofHeader::Unused), Node::Leaf(partial, _) => { let value = self.value .expect( @@ -161,7 +161,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> value is only ever reassigned in the ValueMatch::MatchesLeaf match \ clause, which assigns only to Some" ); - (C::leaf_node(partial.right(), value), ChildRootHeader::Unused) + (C::leaf_node(partial.right(), value), ChildProofHeader::Unused) } Node::Extension(partial, _) => { let child = self.children[0] @@ -170,7 +170,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> partial.right_iter(), partial.len(), child - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) } Node::Branch(_, _) => { let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 4133446e..510109f8 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -31,7 +31,7 @@ use crate::{ TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecComplex, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; -use crate::node_codec::{Bitmap, ChildRootHeader}; +use crate::node_codec::{Bitmap, ChildProofHeader}; use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, rc::Rc, result, vec, vec::Vec, ops::Range, @@ -414,19 +414,19 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { /// /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. - fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, ChildRootHeader) { + fn encode_node(self, register_children: Option<&mut [Option>]>) -> (Vec, ChildProofHeader) { match self.node { Node::Empty => - (C::empty_node().to_vec(), ChildRootHeader::Unused), + (C::empty_node().to_vec(), ChildProofHeader::Unused), Node::Leaf(partial, value) => - (C::leaf_node(partial.right(), value), ChildRootHeader::Unused), + (C::leaf_node(partial.right(), value), ChildProofHeader::Unused), Node::Extension(partial, _) => (C::extension_node( partial.right_iter(), partial.len(), self.children[0] .expect("required by method precondition; qed"), - ), ChildRootHeader::Unused), + ), ChildProofHeader::Unused), Node::Branch(_, value) => if let Some(register_children) = register_children { C::branch_node_common( self.children.into_iter(), @@ -437,7 +437,7 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { (C::branch_node( self.children.into_iter(), value, - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) }, Node::NibbledBranch(partial, _, value) => if let Some(register_children) = register_children { C::branch_node_nibbled_common( @@ -453,7 +453,7 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { partial.len(), self.children.iter(), value, - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) }, } } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index a9cb638a..b2a5e2ab 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -23,7 +23,7 @@ use crate::node_codec::HashDBComplexDyn; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildRootHeader}; +use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildProofHeader}; use crate::nibble::{NibbleVec, NibbleSlice, nibble_ops, BackingByteVec}; use crate::rstd::{ boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, @@ -209,16 +209,16 @@ where self, mut child_cb: F, register_children: Option<&mut [Option>]>, - ) -> (Vec, ChildRootHeader) where + ) -> (Vec, ChildProofHeader) where C: NodeCodecComplex, F: FnMut(NodeHandle, Option<&NibbleSlice>, Option) -> ChildReference, H: Hasher, { match self { - Node::Empty => (C::empty_node().to_vec(), ChildRootHeader::Unused), + Node::Empty => (C::empty_node().to_vec(), ChildProofHeader::Unused), Node::Leaf(partial, value) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - (C::leaf_node(pr.right(), &value), ChildRootHeader::Unused) + (C::leaf_node(pr.right(), &value), ChildProofHeader::Unused) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); @@ -228,7 +228,7 @@ where it, pr.len(), c, - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) }, Node::Branch(mut children, value) => { // map the `NodeHandle`s from the Branch to `ChildReferences` @@ -249,7 +249,7 @@ where (C::branch_node( children, value.as_ref().map(|v| &v[..]), - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) } }, Node::NibbledBranch(partial, mut children, value) => { @@ -281,7 +281,7 @@ where pr.len(), children, value.as_ref().map(|v| &v[..]), - ), ChildRootHeader::Unused) + ), ChildProofHeader::Unused) } }, } From c3da9e66ee006c17f4f96568c14a7ed4388f0ba3 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 11:44:45 +0200 Subject: [PATCH 22/73] Renaming from complex to hybrid. --- memory-db/src/lib.rs | 10 ++-- ordered-trie/src/lib.rs | 18 +++---- test-support/reference-trie/src/lib.rs | 70 +++++++++++++------------- trie-db/src/fatdbmut.rs | 2 +- trie-db/src/iter_build.rs | 64 +++++++++++------------ trie-db/src/lib.rs | 18 +++---- trie-db/src/node_codec.rs | 38 +++++++------- trie-db/src/proof/generate.rs | 20 ++++---- trie-db/src/proof/verify.rs | 40 +++++++-------- trie-db/src/sectriedbmut.rs | 2 +- trie-db/src/trie_codec.rs | 42 ++++++++-------- trie-db/src/triedbmut.rs | 34 ++++++------- 12 files changed, 179 insertions(+), 179 deletions(-) diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index eb280f88..400e24c1 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -19,7 +19,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; -use ordered_trie::{HashDBComplex, HasherComplex}; +use ordered_trie::{HashDBHybrid, HasherHybrid}; use hash_db::{HashDB, HashDBRef, PlainDB, PlainDBRef, Hasher as KeyHasher, AsHashDB, AsPlainDB, Prefix}; use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; @@ -602,13 +602,13 @@ where } } -impl HashDBComplex for MemoryDB +impl HashDBHybrid for MemoryDB where - H: HasherComplex, + H: HasherHybrid, T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, KF: Send + Sync + KeyFunction, { - fn insert_complex< + fn insert_hybrid< I: Iterator>, I2: Iterator, > ( @@ -625,7 +625,7 @@ where return self.hashed_null_node.clone(); } - let key = if let Some(key) = H::hash_complex( + let key = if let Some(key) = H::hash_hybrid( child_proof_header, nb_children, children, diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index cb924a78..e2659da0 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -1494,11 +1494,11 @@ pub trait BinaryHasher: Hasher { type Buffer: AsRef<[u8]> + AsMut<[u8]> + Default; } -pub trait HasherComplex: BinaryHasher { +pub trait HasherHybrid: BinaryHasher { - /// Alternate hash with complex proof allowed + /// Alternate hash with hybrid proof allowed /// TODO expose buffer !! (then memory db use a single buf) - fn hash_complex< + fn hash_hybrid< I: Iterator::Out>>, I2: Iterator::Out>, >( @@ -1510,8 +1510,8 @@ pub trait HasherComplex: BinaryHasher { ) -> Option; } -impl HasherComplex for H { - fn hash_complex< +impl HasherHybrid for H { + fn hash_hybrid< I: Iterator::Out>>, I2: Iterator::Out>, >( @@ -1530,7 +1530,7 @@ impl HasherComplex for H { let iter = children.filter_map(|v| v); // TODO assert same number as count crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof) } else { - // proof node, UsizeKeyNode should be big enough for hasher complex + // proof node, UsizeKeyNode should be big enough for hasher hybrid // case. let iter_key = seq_trie.iter_path_node_key::(None); let iter = children @@ -1561,12 +1561,12 @@ impl HasherComplex for H { } /// Same as HashDB but can modify the value upon storage, and apply -/// `HasherComplex`. -pub trait HashDBComplex: Send + Sync + HashDB { +/// `HasherHybrid`. +pub trait HashDBHybrid: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. - fn insert_complex< + fn insert_hybrid< I: Iterator>, I2: Iterator, >( diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 5ed0f1d6..9d773bd6 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -26,8 +26,8 @@ use trie_db::{ triedbmut::ChildReference, DBValue, trie_visit, - TrieBuilderComplex, - TrieRootComplex, + TrieBuilderHybrid, + TrieRootHybrid, Partial, BinaryHasher, ChildProofHeader, @@ -38,8 +38,8 @@ use std::borrow::Borrow; use keccak_hasher::KeccakHasher; pub use trie_db::{ - decode_compact, encode_compact, HashDBComplexDyn, - nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, NodeCodecComplex, + decode_compact, encode_compact, HashDBHybridDyn, + nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, NodeCodecHybrid, Trie, TrieConfiguration, TrieDB, TrieDBIterator, TrieDBMut, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, TrieMut, Bitmap, BITMAP_LENGTH, }; @@ -53,7 +53,7 @@ pub struct ExtensionLayout; impl TrieLayout for ExtensionLayout { const USE_EXTENSION: bool = true; - const COMPLEX_HASH: bool = true; + const HYBRID_HASH: bool = true; type Hash = KeccakHasher; type Codec = ReferenceNodeCodec; } @@ -66,7 +66,7 @@ pub struct GenericNoExtensionLayout(PhantomData); impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; - const COMPLEX_HASH: bool = true; + const HYBRID_HASH: bool = true; type Hash = H; type Codec = ReferenceNodeCodecNoExt; } @@ -135,7 +135,7 @@ pub fn reference_trie_root_iter_build(input: I) -> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootComplex::::default(); + let mut cb = trie_db::TrieRootHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -145,7 +145,7 @@ pub fn reference_trie_root_no_extension_iter_build(input: I) -> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootComplex::::default(); + let mut cb = trie_db::TrieRootHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -155,7 +155,7 @@ fn reference_trie_root_unhashed_iter_build(input: I) -> Vec where A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootUnhashedComplex::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -165,7 +165,7 @@ fn reference_trie_root_unhashed_no_extension_iter_build(input: I) -> Ve A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootUnhashedComplex::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -642,7 +642,7 @@ impl<'a> Input for ByteSliceInput<'a> { impl ReferenceNodeCodec { fn decode_plan_internal( data: &[u8], - is_complex: bool, + is_hybrid: bool, ) -> ::std::result::Result<(NodePlan, usize), ::Error> { let mut input = ByteSliceInput::new(data); let node = match NodeHeader::decode(&mut input)? { @@ -663,7 +663,7 @@ impl ReferenceNodeCodec { ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - if is_complex { + if is_hybrid { children[i] = Some(NodeHandlePlan::Inline(Range { start: 0, end: 0 })); } else { let count = >::decode(&mut input)?.0 as usize; @@ -780,7 +780,7 @@ impl NodeCodec for ReferenceNodeCodec { } -impl NodeCodecComplex for ReferenceNodeCodec { +impl NodeCodecHybrid for ReferenceNodeCodec { type AdditionalHashesPlan = HashesPlan; fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { @@ -996,7 +996,7 @@ fn encode_proof_internal( impl ReferenceNodeCodecNoExt { fn decode_plan_internal( data: &[u8], - is_complex: bool, + is_hybrid: bool, ) -> ::std::result::Result<(NodePlan, usize), ::Error> { let mut input = ByteSliceInput::new(data); let node = match NodeHeaderNoExt::decode(&mut input)? { @@ -1025,7 +1025,7 @@ impl ReferenceNodeCodecNoExt { ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - if is_complex { + if is_hybrid { children[i] = Some(NodeHandlePlan::Inline(Range { start: 0, end: 0 })); } else { let count = >::decode(&mut input)?.0 as usize; @@ -1205,7 +1205,7 @@ impl ReferenceNodeCodecNoExt { } } -impl NodeCodecComplex for ReferenceNodeCodecNoExt { +impl NodeCodecHybrid for ReferenceNodeCodecNoExt { type AdditionalHashesPlan = HashesPlan; fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { @@ -1272,13 +1272,13 @@ impl NodeCodecComplex for ReferenceNodeCodecNoExt { } /// Compare trie builder and in memory trie. -pub fn compare_implementations + ordered_trie::HashDBComplex + Eq> ( +pub fn compare_implementations + ordered_trie::HashDBHybrid + Eq> ( data: Vec<(Vec, Vec)>, mut memdb: X, mut hashdb: X, ) { let root_new = { - let mut cb = TrieBuilderComplex::new(&mut hashdb); + let mut cb = TrieBuilderHybrid::new(&mut hashdb); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1318,10 +1318,10 @@ pub fn compare_implementations + orde /// Compare trie builder and trie root implementations. pub fn compare_root( data: Vec<(Vec, Vec)>, - mut memdb: impl ordered_trie::HashDBComplex, + mut memdb: impl ordered_trie::HashDBHybrid, ) { let root_new = { - let mut cb = TrieRootComplex::::default(); + let mut cb = TrieRootHybrid::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1342,7 +1342,7 @@ pub fn compare_unhashed( data: Vec<(Vec, Vec)>, ) { let root_new = { - let mut cb = trie_db::TrieRootUnhashedComplex::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1357,7 +1357,7 @@ pub fn compare_unhashed_no_extension( data: Vec<(Vec, Vec)>, ) { let root_new = { - let mut cb = trie_db::TrieRootUnhashedComplex::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1375,7 +1375,7 @@ pub fn calc_root( A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRootComplex::::default(); + let mut cb = TrieRootHybrid::::default(); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1390,7 +1390,7 @@ pub fn calc_root_no_extension( A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRootComplex::::default(); + let mut cb = TrieRootHybrid::::default(); trie_db::trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1404,9 +1404,9 @@ pub fn calc_root_build( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + ordered_trie::HashDBComplex, + DB: hash_db::HashDB + ordered_trie::HashDBHybrid, { - let mut cb = TrieBuilderComplex::new(hashdb); + let mut cb = TrieBuilderHybrid::new(hashdb); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1421,9 +1421,9 @@ pub fn calc_root_build_no_extension( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + ordered_trie::HashDBComplex, + DB: hash_db::HashDB + ordered_trie::HashDBHybrid, { - let mut cb = TrieBuilderComplex::new(hashdb); + let mut cb = TrieBuilderHybrid::new(hashdb); trie_db::trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1432,11 +1432,11 @@ pub fn calc_root_build_no_extension( /// This uses the variant without extension nodes. pub fn compare_implementations_no_extension( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, - mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, + mut memdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, + mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, ) { let root_new = { - let mut cb = TrieBuilderComplex::new(&mut hashdb); + let mut cb = TrieBuilderHybrid::new(&mut hashdb); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1475,8 +1475,8 @@ pub fn compare_implementations_no_extension( /// ordering before running when trie_build expect correct ordering). pub fn compare_implementations_no_extension_unordered( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, - mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBComplex, + mut memdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, + mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, ) { let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { @@ -1489,7 +1489,7 @@ pub fn compare_implementations_no_extension_unordered( t.root().clone() }; let root_new = { - let mut cb = TrieBuilderComplex::new(&mut hashdb); + let mut cb = TrieBuilderHybrid::new(&mut hashdb); trie_visit::(b_map.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1520,7 +1520,7 @@ pub fn compare_implementations_no_extension_unordered( /// its input test data. pub fn compare_no_extension_insert_remove( data: Vec<(bool, Vec, Vec)>, - mut memdb: impl ordered_trie::HashDBComplex, + mut memdb: impl ordered_trie::HashDBHybrid, ) { let mut data2 = std::collections::BTreeMap::new(); let mut root = Default::default(); diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs index 714e1b20..7a53177a 100644 --- a/trie-db/src/fatdbmut.rs +++ b/trie-db/src/fatdbmut.rs @@ -13,7 +13,7 @@ // limitations under the License. use hash_db::{Hasher, EMPTY_PREFIX}; -use crate::node_codec::HashDBComplexDyn as HashDB; +use crate::node_codec::HashDBHybridDyn as HashDB; use super::{Result, DBValue, TrieDBMut, TrieMut, TrieLayout, TrieHash, CError}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index d8e128e8..70f91780 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -18,12 +18,12 @@ //! See `trie_visit` function. use hash_db::{Hasher, HashDB, Prefix}; -use ordered_trie::{HashDBComplex, HasherComplex}; +use ordered_trie::{HashDBHybrid, HasherHybrid}; use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range}; use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; use crate::nibble::nibble_ops; -use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildProofHeader}; +use crate::node_codec::{NodeCodec, NodeCodecHybrid, ChildProofHeader}; use crate::{TrieLayout, TrieHash}; use crate::rstd::borrow::Borrow; @@ -54,7 +54,7 @@ const INITIAL_DEPTH: usize = 10; #[inline] fn register_children_buf() -> Option<[Option>; 16]> { - if T::COMPLEX_HASH { + if T::HYBRID_HASH { Some(Default::default()) } else { None @@ -222,7 +222,7 @@ impl CacheAccum ), ChildProofHeader::Unused) }; let pr = NibbleSlice::new_offset(&key_branch, branch_d); - let branch_hash = if T::COMPLEX_HASH { + let branch_hash = if T::HYBRID_HASH { let len = self.0[last].0.as_ref().iter().filter(|v| v.is_some()).count(); let children = self.0[last].0.as_ref().iter(); callback.process(pr.left(), encoded, is_root && nkey.is_none(), Some((children, len))) @@ -279,7 +279,7 @@ impl CacheAccum &key_branch, branch_d - ext_length, ); - let result = if T::COMPLEX_HASH { + let result = if T::HYBRID_HASH { let len = self.0[last].0.as_ref().iter().filter(|v| v.is_some()).count(); let children = self.0[last].0.as_ref().iter(); callback.process(pr.left(), encoded, is_root, Some((children, len))) @@ -374,7 +374,7 @@ pub trait ProcessEncodedNode { prefix: Prefix, encoded_node: (Vec, ChildProofHeader), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference; } @@ -396,15 +396,15 @@ impl<'a, H, HO, V, DB> TrieBuilder<'a, H, HO, V, DB> { /// Get trie root and insert visited node in a hash_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). -pub struct TrieBuilderComplex<'a, H, HO, V, DB> { +pub struct TrieBuilderHybrid<'a, H, HO, V, DB> { db: &'a mut DB, pub root: Option, _ph: PhantomData<(H, V)>, } -impl<'a, H, HO, V, DB> TrieBuilderComplex<'a, H, HO, V, DB> { +impl<'a, H, HO, V, DB> TrieBuilderHybrid<'a, H, HO, V, DB> { pub fn new(db: &'a mut DB) -> Self { - TrieBuilderComplex { db, root: None, _ph: PhantomData } + TrieBuilderHybrid { db, root: None, _ph: PhantomData } } } @@ -417,7 +417,7 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, // TODO different trait?? - _complex_hash: Option<(impl Iterator>>>, usize)>, + _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -434,14 +434,14 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> } } -impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode<::Out> - for TrieBuilderComplex<'a, H, ::Out, V, DB> { +impl<'a, H: HasherHybrid, V, DB: HashDBHybrid> ProcessEncodedNode<::Out> + for TrieBuilderHybrid<'a, H, ::Out, V, DB> { fn process( &mut self, prefix: Prefix, (encoded_node, common): (Vec, ChildProofHeader), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -451,14 +451,14 @@ impl<'a, H: HasherComplex, V, DB: HashDBComplex> ProcessEncodedNode< Some(Some(v.clone())), Some(ChildReference::Inline(v, _l)) => Some(Some(v.clone())), None => None, }); - self.db.insert_complex( + self.db.insert_hybrid( prefix, &encoded_node[..], common.header(&encoded_node[..]), @@ -497,7 +497,7 @@ impl ProcessEncodedNode<::Out> for TrieRoot, ChildProofHeader), is_root: bool, // TODO different trait - _complex_hash: Option<(impl Iterator>>>, usize)>, + _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -515,25 +515,25 @@ impl ProcessEncodedNode<::Out> for TrieRoot { +pub struct TrieRootHybrid { /// The resulting root. pub root: Option, _ph: PhantomData, } -impl Default for TrieRootComplex { +impl Default for TrieRootHybrid { fn default() -> Self { - TrieRootComplex { root: None, _ph: PhantomData } + TrieRootHybrid { root: None, _ph: PhantomData } } } -impl ProcessEncodedNode<::Out> for TrieRootComplex::Out> { +impl ProcessEncodedNode<::Out> for TrieRootHybrid::Out> { fn process( &mut self, _: Prefix, (encoded_node, common): (Vec, ChildProofHeader), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -542,14 +542,14 @@ impl ProcessEncodedNode<::Out> for TrieRootComple return ChildReference::Inline(h, len); } - let hash = if let Some((children, nb_children)) = complex_hash { + let hash = if let Some((children, nb_children)) = hybrid_hash { let iter = children .filter_map(|v| match v.borrow().as_ref() { Some(ChildReference::Hash(v)) => Some(Some(v.clone())), Some(ChildReference::Inline(v, _l)) => Some(Some(v.clone())), None => None, }); - ::hash_complex( + ::hash_hybrid( common.header(&encoded_node[..]), nb_children, iter, @@ -582,15 +582,15 @@ impl Default for TrieRootUnhashed { } /// Get the trie root node encoding. -pub struct TrieRootUnhashedComplex { +pub struct TrieRootUnhashedHybrid { /// The resulting encoded root. pub root: Option>, _ph: PhantomData, } -impl Default for TrieRootUnhashedComplex { +impl Default for TrieRootUnhashedHybrid { fn default() -> Self { - TrieRootUnhashedComplex { root: None, _ph: PhantomData } + TrieRootUnhashedHybrid { root: None, _ph: PhantomData } } } @@ -618,7 +618,7 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, ChildProofHeader), is_root: bool, // TODO different trait? - _complex_hash: Option<(impl Iterator>>>, usize)>, + _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { println!("Encoded node: {:x?}", &encoded_node); println!(" with prefix: {:x?}", &p); @@ -646,7 +646,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, // TODO different trait - _complex_hash: Option<(impl Iterator>>>, usize)>, + _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -663,13 +663,13 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { } } -impl ProcessEncodedNode<::Out> for TrieRootUnhashedComplex { +impl ProcessEncodedNode<::Out> for TrieRootUnhashedHybrid { fn process( &mut self, _: Prefix, (encoded_node, common): (Vec, ChildProofHeader), is_root: bool, - complex_hash: Option<(impl Iterator>>>, usize)>, + hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { @@ -678,14 +678,14 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhash return ChildReference::Inline(h, len); } - let hash = if let Some((children, nb_children)) = complex_hash { + let hash = if let Some((children, nb_children)) = hybrid_hash { let iter = children .filter_map(|v| match v.borrow().as_ref() { Some(ChildReference::Hash(v)) => Some(Some(v.clone())), Some(ChildReference::Inline(v, _l)) => Some(Some(v.clone())), None => None, }); - ::hash_complex( + ::hash_hybrid( common.header(&encoded_node[..]), nb_children, iter, diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 2dd73fa8..ff9943f4 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -70,10 +70,10 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; -pub use crate::node_codec::{NodeCodec, NodeCodecComplex, Partial, HashDBComplexDyn, ChildProofHeader, +pub use crate::node_codec::{NodeCodec, NodeCodecHybrid, Partial, HashDBHybridDyn, ChildProofHeader, Bitmap, BITMAP_LENGTH, HashesPlan}; -pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedComplex, - TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootComplex, TrieBuilderComplex}; +pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybrid, + TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; pub use crate::trie_codec::{decode_compact, encode_compact, binary_additional_hashes}; pub use ordered_trie::BinaryHasher; @@ -349,7 +349,7 @@ where /// Create new mutable instance of Trie. pub fn create( &self, - db: &'db mut dyn HashDBComplexDyn, + db: &'db mut dyn HashDBHybridDyn, root: &'db mut TrieHash, ) -> Box + 'db> { match self.spec { @@ -362,7 +362,7 @@ where /// Create new mutable instance of trie and check for errors. pub fn from_existing( &self, - db: &'db mut dyn HashDBComplexDyn, + db: &'db mut dyn HashDBHybridDyn, root: &'db mut TrieHash, ) -> Result + 'db>, TrieHash, CError> { match self.spec { @@ -384,15 +384,15 @@ pub trait TrieLayout { /// no partial in branch, if false the trie will only /// use branch and node with partials in both. const USE_EXTENSION: bool; - /// Does the layout implement a complex hash. - /// Note that if does not, the `NodeCodecComplex` hash + /// Does the layout implement a hybrid hash. + /// Note that if does not, the `NodeCodecHybrid` hash /// associated codec only really need to implement `NodeCodec` /// and dummy implementation can be used. - const COMPLEX_HASH: bool; + const HYBRID_HASH: bool; /// Hasher to use for this trie. type Hash: BinaryHasher; /// Codec to use (needs to match hasher and nibble ops). - type Codec: NodeCodecComplex::Out>; + type Codec: NodeCodecHybrid::Out>; } /// This trait associates a trie definition with preferred methods. diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index f784190f..853a1ce1 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -86,13 +86,13 @@ pub trait NodeCodec: Sized { ) -> Vec; } -/// Trait for handling complex proof. +/// Trait for handling hybrid proof. /// This adds methods to basic node codec in order to support: /// - storage encoding with existing `NodeCodec` methods /// - encode a proof specific representation. (usually the common representation and /// the merkle proof of the children stored encoded hash). /// - Intermediate optional common representation shared between storage -pub trait NodeCodecComplex: NodeCodec { +pub trait NodeCodecHybrid: NodeCodec { /// Sequence of hashes needed for the children proof verification. type AdditionalHashesPlan: Iterator>; @@ -179,7 +179,7 @@ pub trait NodeCodecComplex: NodeCodec { /// The node hash is the hash of these information and the merkle root of its children. #[derive(Clone)] pub enum ChildProofHeader { - /// No need for complex hash. + /// No need for hybrid hash. Unused, /// Range over the branch encoded for storage. Range(Range), @@ -197,17 +197,17 @@ impl ChildProofHeader { } } -use ordered_trie::{HashDBComplex, HasherComplex}; +use ordered_trie::{HashDBHybrid, HasherHybrid}; use hash_db::{HashDB, Prefix, HashDBRef, Hasher}; -pub trait HashDBComplexDyn: Send + Sync + HashDB { +pub trait HashDBHybridDyn: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. /// - /// TODO warn semantic of children differs from HashDBComplex (in HashDBComplex it is the + /// TODO warn semantic of children differs from HashDBHybrid (in HashDBHybrid it is the /// children of the binary hasher, here it is the children of the patricia merkle trie). - fn insert_complex( + fn insert_hybrid( &mut self, prefix: Prefix, value: &[u8], @@ -216,8 +216,8 @@ pub trait HashDBComplexDyn: Send + Sync + HashDB { ) -> H::Out; } -impl> HashDBComplexDyn for C { - fn insert_complex( +impl> HashDBHybridDyn for C { + fn insert_hybrid( &mut self, prefix: Prefix, value: &[u8], @@ -227,12 +227,12 @@ impl> HashDBComplexDyn for C { // TODO factor this with iter_build (just use the trait) let nb_children = children.iter().filter(|v| v.is_some()).count(); - let children = ComplexLayoutIterValues::new( + let children = HybridLayoutIterValues::new( children.iter().filter_map(|v| v.as_ref()), value, ); - >::insert_complex( + >::insert_hybrid( self, prefix, value, @@ -245,7 +245,7 @@ impl> HashDBComplexDyn for C { } } -impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDBComplexDyn { +impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDBHybridDyn { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { self.as_hash_db().get(key, prefix) } @@ -255,7 +255,7 @@ impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDBComplexDyn { } } -impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDBComplexDyn { +impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDBHybridDyn { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { self.as_hash_db().get(key, prefix) } @@ -267,20 +267,20 @@ impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDBComplexDyn { // TODO this using a buffer is bad (we should switch // binary hasher to use slice as input (or be able to)) -pub struct ComplexLayoutIterValues<'a, HO, I> { +pub struct HybridLayoutIterValues<'a, HO, I> { children: I, node: &'a [u8], _ph: PhantomData, } /* code snippet for children iter: -ComplexLayoutIterValues::new(nb_children, children, value) +HybridLayoutIterValues::new(nb_children, children, value) .map(|(is_defined, v)| { debug_assert!(is_defined); v }); code snippet for proof - let iter = ComplexLayoutIterValues::new(nb_children, children, value) + let iter = HybridLayoutIterValues::new(nb_children, children, value) .zip(iter_key) .filter_map(|((is_defined, hash), key)| if is_defined { Some((key, hash)) @@ -289,9 +289,9 @@ code snippet for proof }); */ -impl<'a, HO: Default, I> ComplexLayoutIterValues<'a, HO, I> { +impl<'a, HO: Default, I> HybridLayoutIterValues<'a, HO, I> { pub fn new(children: I, node: &'a[u8]) -> Self { - ComplexLayoutIterValues { + HybridLayoutIterValues { children, node, _ph: PhantomData, @@ -299,7 +299,7 @@ impl<'a, HO: Default, I> ComplexLayoutIterValues<'a, HO, I> { } } -impl<'a, HO: AsMut<[u8]> + Default, I: Iterator>> Iterator for ComplexLayoutIterValues<'a, HO, I> { +impl<'a, HO: AsMut<[u8]> + Default, I: Iterator>> Iterator for HybridLayoutIterValues<'a, HO, I> { type Item = Option; fn next(&mut self) -> Option { diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index bd53fb15..e042b89d 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -23,7 +23,7 @@ use hash_db::Hasher; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, Result as TrieResult, Trie, TrieError, TrieHash, - TrieLayout, NodeCodecComplex, + TrieLayout, NodeCodecHybrid, }; use ordered_trie::BinaryHasher; @@ -46,7 +46,7 @@ struct StackEntry<'a, C: NodeCodec, H> { _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> +impl<'a, C: NodeCodecHybrid, H: BinaryHasher> StackEntry<'a, C, H> where H: BinaryHasher, { @@ -83,7 +83,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> /// Encode this entry to an encoded trie node with data properly omitted. fn encode_node( mut self, - complex: bool, + hybrid: bool, hash_buf: &mut H::Buffer, ) -> TrieResult, C::HashOut, C::Error> { let node_data = self.node.data(); @@ -116,7 +116,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> self.child_index, &mut self.children, )?; - if !self.is_inline && complex { + if !self.is_inline && hybrid { let hash_proof_header = C::branch_node_for_hash( self.children.iter(), value_with_omission(node_data, value, self.omit_value), @@ -141,7 +141,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> self.child_index, &mut self.children )?; - if !self.is_inline && complex { + if !self.is_inline && hybrid { let hash_proof_header = C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), @@ -284,7 +284,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) let key = LeftNibbleSlice::new(key_bytes); // Unwind the stack until the new entry is a child of the last entry on the stack. - unwind_stack(&mut stack, &mut proof_nodes, Some(&key), L::COMPLEX_HASH, hash_buf)?; + unwind_stack(&mut stack, &mut proof_nodes, Some(&key), L::HYBRID_HASH, hash_buf)?; // Perform the trie lookup for the next key, recording the sequence of nodes traversed. let mut recorder = Recorder::new(); @@ -392,7 +392,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) } } - unwind_stack(&mut stack, &mut proof_nodes, None, L::COMPLEX_HASH, hash_buf)?; + unwind_stack(&mut stack, &mut proof_nodes, None, L::HYBRID_HASH, hash_buf)?; Ok(proof_nodes) } @@ -538,11 +538,11 @@ fn value_with_omission<'a>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( +fn unwind_stack( stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, - complex: bool, + hybrid: bool, hash_buf: &mut H::Buffer, ) -> TrieResult<(), C::HashOut, C::Error> where @@ -558,7 +558,7 @@ fn unwind_stack( _ => { // Pop and finalize node from the stack. let index = entry.output_index; - let encoded = entry.encode_node(complex, hash_buf)?; + let encoded = entry.encode_node(hybrid, hash_buf)?; if let Some(parent_entry) = stack.last_mut() { parent_entry.set_child(&encoded); } diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index a8f84dca..f0c29808 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -17,10 +17,10 @@ use crate::rstd::{ }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{Node, NodeHandle}, NodeCodecComplex, TrieHash, TrieLayout, ChildProofHeader, + node::{Node, NodeHandle}, NodeCodecHybrid, TrieHash, TrieLayout, ChildProofHeader, }; use hash_db::Hasher; -use ordered_trie::{BinaryHasher, HasherComplex}; +use ordered_trie::{BinaryHasher, HasherHybrid}; use crate::node_codec::{Bitmap, HashesIter}; @@ -96,7 +96,7 @@ impl std::error::Error for } } -struct StackEntry<'a, C: NodeCodecComplex, H> { +struct StackEntry<'a, C: NodeCodecHybrid, H> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, node: Node<'a>, @@ -108,20 +108,20 @@ struct StackEntry<'a, C: NodeCodecComplex, H> { child_index: usize, /// The child references to use in reconstructing the trie nodes. children: Vec>>, - /// Proof info if a complex proof is needed. - complex: Option<(Bitmap, HashesIter<'a, C::AdditionalHashesPlan, C::HashOut>)>, + /// Proof info if a hybrid proof is needed. + hybrid: Option<(Bitmap, HashesIter<'a, C::AdditionalHashesPlan, C::HashOut>)>, _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> +impl<'a, C: NodeCodecHybrid, H: BinaryHasher> StackEntry<'a, C, H> where H: BinaryHasher, { - fn new(node_data: &'a [u8], prefix: LeftNibbleSlice<'a>, is_inline: bool, complex: bool) + fn new(node_data: &'a [u8], prefix: LeftNibbleSlice<'a>, is_inline: bool, hybrid: bool) -> Result> { let children = vec![None; NIBBLE_LENGTH]; // TODO use array - let (node, complex) = if !is_inline && complex { + let (node, hybrid) = if !is_inline && hybrid { // TODO factorize with trie_codec let encoded_node = node_data; C::decode_proof(encoded_node) @@ -143,7 +143,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> value, child_index: 0, children, - complex, + hybrid, _marker: PhantomData::default(), }) } @@ -199,7 +199,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> &mut self, child_prefix: LeftNibbleSlice<'a>, proof_iter: &mut I, - complex: bool, + hybrid: bool, ) -> Result> where I: Iterator>, @@ -208,7 +208,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> Node::Extension(_, child) => { // Guaranteed because of sorted keys order. assert_eq!(self.child_index, 0); - Self::make_child_entry(proof_iter, child, child_prefix, complex) + Self::make_child_entry(proof_iter, child, child_prefix, hybrid) } Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { // because this is a branch @@ -226,7 +226,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> } let child = children[self.child_index] .expect("guaranteed by advance_item"); - Self::make_child_entry(proof_iter, child, child_prefix, complex) + Self::make_child_entry(proof_iter, child, child_prefix, hybrid) } _ => panic!("cannot have children"), } @@ -260,7 +260,7 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> proof_iter: &mut I, child: NodeHandle<'a>, prefix: LeftNibbleSlice<'a>, - complex: bool, + hybrid: bool, ) -> Result> where I: Iterator>, @@ -270,9 +270,9 @@ impl<'a, C: NodeCodecComplex, H: BinaryHasher> StackEntry<'a, C, H> if data.is_empty() { let node_data = proof_iter.next() .ok_or(Error::IncompleteProof)?; - StackEntry::new(node_data, prefix, false, complex) + StackEntry::new(node_data, prefix, false, hybrid) } else { - StackEntry::new(data, prefix, true, complex) + StackEntry::new(data, prefix, true, hybrid) } } NodeHandle::Hash(data) => { @@ -455,7 +455,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve root_node, LeftNibbleSlice::new(&[]), false, - L::COMPLEX_HASH, + L::HYBRID_HASH, )?; loop { // Insert omitted value. @@ -464,7 +464,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve let next_entry = last_entry.advance_child_index( child_prefix, &mut proof_iter, - L::COMPLEX_HASH, + L::HYBRID_HASH, )?; stack.push(last_entry); last_entry = next_entry; @@ -481,7 +481,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve &mut hash.as_mut()[..node_data.len()].copy_from_slice(node_data.as_ref()); ChildReference::Inline(hash, node_data.len()) } else { - ChildReference::Hash(if let Some((bitmap_keys, additional_hash)) = last_entry.complex { + ChildReference::Hash(if let Some((bitmap_keys, additional_hash)) = last_entry.hybrid { let children = last_entry.children; let nb_children = children.iter().filter(|v| v.is_some()).count(); let children = children.into_iter() @@ -503,7 +503,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } }); - if let Some(h) = L::Hash::hash_complex( + if let Some(h) = L::Hash::hash_hybrid( &common.header(node_data.as_slice())[..], nb_children, children, @@ -513,7 +513,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve h } else { // TODO better error for the invalid - // complex hash + // hybrid hash return Err(Error::RootMismatch(Default::default())); } } else { diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs index aeff2cef..0e94134c 100644 --- a/trie-db/src/sectriedbmut.rs +++ b/trie-db/src/sectriedbmut.rs @@ -13,7 +13,7 @@ // limitations under the License. use hash_db::{Hasher}; -use crate::node_codec::HashDBComplexDyn as HashDB; +use crate::node_codec::HashDBHybridDyn as HashDB; use super::{Result, DBValue, TrieMut, TrieDBMut, TrieLayout, TrieHash, CError}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 510109f8..3af081fa 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -25,10 +25,10 @@ //! expected to save roughly (n - 1) hashes in size where n is the number of nodes in the partial //! trie. -pub use ordered_trie::{BinaryHasher, HashDBComplex}; +pub use ordered_trie::{BinaryHasher, HashDBHybrid}; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, - TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecComplex, + TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecHybrid, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, }; use crate::node_codec::{Bitmap, ChildProofHeader}; @@ -52,7 +52,7 @@ struct EncoderStackEntry { _marker: PhantomData, } -impl EncoderStackEntry { +impl EncoderStackEntry { /// Given the prefix of the next child node, identify its index and advance `child_index` to /// that. For a given entry, this must be called sequentially only with strictly increasing /// child prefixes. Returns an error if the child prefix is not a child of this entry or if @@ -99,7 +99,7 @@ impl EncoderStackEntry { /// Generates the encoding of the subtrie rooted at this entry. fn encode_node( &self, - complex_hash: bool, + hybrid_hash: bool, hash_buf: &mut H::Buffer, ) -> Result, C::HashOut, C::Error> where @@ -119,7 +119,7 @@ impl EncoderStackEntry { }, NodePlan::Branch { value, children } => { let children = Self::branch_children(node_data, &children, &self.omit_children[..])?; - if complex_hash { + if hybrid_hash { let hash_proof_header = C::branch_node_for_hash( children.iter(), value.clone().map(|range| &node_data[range]), @@ -139,7 +139,7 @@ impl EncoderStackEntry { NodePlan::NibbledBranch { partial, value, children } => { let children = Self::branch_children(node_data, &children, &self.omit_children[..])?; let partial = partial.build(node_data); - if complex_hash { + if hybrid_hash { let hash_proof_header = C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), @@ -208,7 +208,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE { let mut output = Vec::new(); - // TODO make it optional and replace boolean is_complex + // TODO make it optional and replace boolean is_hybrid let mut hash_buf = ::Buffer::default(); let hash_buf = &mut hash_buf; @@ -254,7 +254,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE break; } else { output[last_entry.output_index] = last_entry.encode_node::( - L::COMPLEX_HASH, + L::HYBRID_HASH, hash_buf, )?; } @@ -284,7 +284,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE while let Some(entry) = stack.pop() { output[entry.output_index] = entry.encode_node::( - L::COMPLEX_HASH, + L::HYBRID_HASH, hash_buf, )?; } @@ -300,12 +300,12 @@ struct DecoderStackEntry<'a, C: NodeCodec, F> { /// The reconstructed child references. /// TODO remove Vec here!!! children: Vec>>, - /// Complex proof input - complex: Option<(Bitmap, F)>, + /// Hybrid proof input + hybrid: Option<(Bitmap, F)>, _marker: PhantomData, } -impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { +impl<'a, C: NodeCodecHybrid, F> DecoderStackEntry<'a, C, F> { /// Advance the child index until either it exceeds the number of children or the child is /// marked as omitted. Omitted children are indicated by an empty inline reference. For each /// child that is passed over and not omitted, copy over the child reference from the node to @@ -331,7 +331,7 @@ impl<'a, C: NodeCodecComplex, F> DecoderStackEntry<'a, C, F> { self.child_index += 1; } Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { - if let Some((bitmap_keys, _)) = self.complex.as_ref() { + if let Some((bitmap_keys, _)) = self.hybrid.as_ref() { while self.child_index < NIBBLE_LENGTH { match children[self.child_index] { Some(NodeHandle::Inline(data)) if data.is_empty() @@ -475,7 +475,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDBComplex, + DB: HashDBHybrid, { // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. @@ -485,7 +485,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) let mut prefix = NibbleVec::new(); for (i, encoded_node) in encoded.iter().enumerate() { - let (node, complex) = if L::COMPLEX_HASH { + let (node, hybrid) = if L::HYBRID_HASH { L::Codec::decode_proof(encoded_node) .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))? } else { @@ -504,7 +504,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) node, child_index: 0, children: vec![None; children_len], - complex, + hybrid, _marker: PhantomData::default(), }; @@ -516,18 +516,18 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) } let mut register_children: [Option<_>; NIBBLE_LENGTH]; - let mut register_children = if last_entry.complex.is_some() { + let mut register_children = if last_entry.hybrid.is_some() { register_children = Default::default(); Some(&mut register_children[..]) } else { None }; - let complex = last_entry.complex.take(); + let hybrid = last_entry.hybrid.take(); // Since `advance_child_index` returned true, the preconditions for `encode_node` are // satisfied. let (node_data, common) = last_entry.encode_node(register_children.as_mut().map(|r| r.as_mut())); - let node_hash = if let Some((_bitmap_keys, additional_hashes)) = complex { - let children = register_children.expect("Set to some if complex"); + let node_hash = if let Some((_bitmap_keys, additional_hashes)) = hybrid { + let children = register_children.expect("Set to some if hybrid"); let nb_children = children.iter().filter(|v| v.is_some()).count(); let children = children.iter() .filter_map(|v| v.clone()) @@ -544,7 +544,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) None } }); - db.insert_complex( + db.insert_hybrid( prefix.as_prefix(), &node_data[..], common.header(&node_data[..]), diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index b2a5e2ab..01e340ba 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -19,11 +19,11 @@ use super::{Result, TrieError, TrieMut, TrieLayout, TrieHash, CError}; use super::lookup::Lookup; use super::node::{NodeHandle as EncodedNodeHandle, Node as EncodedNode, decode_hash}; -use crate::node_codec::HashDBComplexDyn; +use crate::node_codec::HashDBHybridDyn; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::node_codec::{NodeCodec, NodeCodecComplex, ChildProofHeader}; +use crate::node_codec::{NodeCodec, NodeCodecHybrid, ChildProofHeader}; use crate::nibble::{NibbleVec, NibbleSlice, nibble_ops, BackingByteVec}; use crate::rstd::{ boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, @@ -126,7 +126,7 @@ where fn inline_or_hash( parent_hash: H::Out, child: EncodedNodeHandle, - db: &dyn HashDBComplexDyn, + db: &dyn HashDBHybridDyn, storage: &mut NodeStorage ) -> Result, H::Out, C::Error> where @@ -151,7 +151,7 @@ where fn from_encoded<'a, 'b, C, H>( node_hash: H::Out, data: &'a[u8], - db: &dyn HashDBComplexDyn, + db: &dyn HashDBHybridDyn, storage: &'b mut NodeStorage, ) -> Result where @@ -210,7 +210,7 @@ where mut child_cb: F, register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader) where - C: NodeCodecComplex, + C: NodeCodecHybrid, F: FnMut(NodeHandle, Option<&NibbleSlice>, Option) -> ChildReference, H: Hasher, { @@ -411,7 +411,7 @@ impl<'a, H> Index<&'a StorageHandle> for NodeStorage { } } -/// A `Trie` implementation using a generic `HashDBComplexDyn` backing database. +/// A `Trie` implementation using a generic `HashDBHybridDyn` backing database. /// /// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. /// Note that changes are not committed to the database until `commit` is called. @@ -443,7 +443,7 @@ where L: TrieLayout, { storage: NodeStorage>, - db: &'a mut dyn HashDBComplexDyn, + db: &'a mut dyn HashDBHybridDyn, root: &'a mut TrieHash, root_handle: NodeHandle>, death_row: HashSet<(TrieHash, (BackingByteVec, Option))>, @@ -459,7 +459,7 @@ where #[inline] fn register_children_buf(node: &Node>) -> Option<[Option>; 16]> { match node { - Node::NibbledBranch(..) | Node::Branch(..) => if L::COMPLEX_HASH { + Node::NibbledBranch(..) | Node::Branch(..) => if L::HYBRID_HASH { Some(Default::default()) } else { None @@ -469,7 +469,7 @@ where } /// Create a new trie with backing database `db` and empty `root`. - pub fn new(db: &'a mut dyn HashDBComplexDyn, root: &'a mut TrieHash) -> Self { + pub fn new(db: &'a mut dyn HashDBHybridDyn, root: &'a mut TrieHash) -> Self { *root = L::Codec::hashed_null_node(); let root_handle = NodeHandle::Hash(L::Codec::hashed_null_node()); @@ -486,7 +486,7 @@ where /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. pub fn from_existing( - db: &'a mut dyn HashDBComplexDyn, + db: &'a mut dyn HashDBHybridDyn, root: &'a mut TrieHash, ) -> Result, CError> { if !db.contains(root, EMPTY_PREFIX) { @@ -504,12 +504,12 @@ where }) } /// Get the backing database. - pub fn db(&self) -> &dyn HashDBComplexDyn { + pub fn db(&self) -> &dyn HashDBHybridDyn { self.db } /// Get the backing database mutably. - pub fn db_mut(&mut self) -> &mut dyn HashDBComplexDyn { + pub fn db_mut(&mut self) -> &mut dyn HashDBHybridDyn { self.db } @@ -1473,7 +1473,7 @@ where #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:#x?}", &encoded_root[..]); if let Some(children) = register_children { - *self.root = self.db.insert_complex( + *self.root = self.db.insert_hybrid( EMPTY_PREFIX, &encoded_root[..], &children[..], @@ -1531,7 +1531,7 @@ where }; if encoded.len() >= L::Hash::LENGTH { let hash = if let Some(children) = register_children { - self.db.insert_complex( + self.db.insert_hybrid( prefix.as_prefix(), &encoded[..], &children[..], @@ -1679,13 +1679,13 @@ mod tests { use memory_db::{MemoryDB, PrefixedKey}; use hash_db::Hasher; use keccak_hasher::KeccakHasher; - use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, HashDBComplexDyn, + use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, reference_trie_root_no_extension_iter_build as reference_trie_root_no_extension}; use crate::nibble::BackingByteVec; fn populate_trie<'db>( - db: &'db mut dyn HashDBComplexDyn, + db: &'db mut dyn HashDBHybridDyn, root: &'db mut ::Out, v: &[(Vec, Vec)] ) -> RefTrieDBMut<'db> { @@ -1706,7 +1706,7 @@ mod tests { } fn populate_trie_no_extension<'db>( - db: &'db mut dyn HashDBComplexDyn, + db: &'db mut dyn HashDBHybridDyn, root: &'db mut ::Out, v: &[(Vec, Vec)] ) -> RefTrieDBMutNoExt<'db> { From 5154c642b84109d8f648b9e166e45c3d828c97af Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 11:52:58 +0200 Subject: [PATCH 23/73] codec decode renaming (indicates it is related to compact proof). --- test-support/reference-trie/src/lib.rs | 10 +++--- trie-db/src/node_codec.rs | 42 +++++++++++++------------- trie-db/src/proof/verify.rs | 2 +- trie-db/src/trie_codec.rs | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 9d773bd6..a2e27169 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -783,9 +783,9 @@ impl NodeCodec for ReferenceNodeCodec { impl NodeCodecHybrid for ReferenceNodeCodec { type AdditionalHashesPlan = HashesPlan; - fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { + fn decode_plan_compact_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { let (node, offset) = Self::decode_plan_internal(data, true)?; - decode_plan_proof_internal(data, offset, node, H::LENGTH) + decode_plan_compact_proof_internal(data, offset, node, H::LENGTH) } fn branch_node_common( @@ -906,7 +906,7 @@ impl ReferenceNodeCodec { } } -fn decode_plan_proof_internal( +fn decode_plan_compact_proof_internal( data: &[u8], mut offset: usize, mut node: NodePlan, @@ -1208,9 +1208,9 @@ impl ReferenceNodeCodecNoExt { impl NodeCodecHybrid for ReferenceNodeCodecNoExt { type AdditionalHashesPlan = HashesPlan; - fn decode_plan_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { + fn decode_plan_compact_proof(data: &[u8]) -> Result<(NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>), Self::Error> { let (node, offset) = Self::decode_plan_internal(data, true)?; - decode_plan_proof_internal(data, offset, node, H::LENGTH) + decode_plan_compact_proof_internal(data, offset, node, H::LENGTH) } fn branch_node_common( diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 853a1ce1..e652a374 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -96,8 +96,8 @@ pub trait NodeCodecHybrid: NodeCodec { /// Sequence of hashes needed for the children proof verification. type AdditionalHashesPlan: Iterator>; - /// Technical function to implement `decode_proof`. - fn decode_plan_proof(data: &[u8]) -> Result<( + /// Technical function to implement `decode_compact_proof`. + fn decode_plan_compact_proof(data: &[u8]) -> Result<( NodePlan, Option<(Bitmap, Self::AdditionalHashesPlan)>, ), Self::Error>; @@ -111,15 +111,32 @@ pub trait NodeCodecHybrid: NodeCodec { /// verification. /// In practice this contains inline node (inline nodes are always added /// to the proof) and ommitted children hash (compacted hash). - fn decode_proof(data: &[u8]) -> Result<( + fn decode_compact_proof(data: &[u8]) -> Result<( Node, Option<(Bitmap, HashesIter)>, ), Self::Error> { - let (plan, hashes) = Self::decode_plan_proof(data)?; + let (plan, hashes) = Self::decode_plan_compact_proof(data)?; let hashes = hashes.map(|(bitmap, hashes)| (bitmap, HashesIter::new(data, hashes))); Ok((plan.build(data), hashes)) } + /// Build compact proof encoding from branch info. + /// + /// - `hash_proof_header`: the part common with the header info from hash. + /// It can be calculated from `branch_node_common` through + /// `ChildProofHeader` call, or directly by `branch_node_for_hash`. + /// - `children`: contains all children, with compact (ommited children) defined as + /// a null length inline node. + /// The children to be include in the proof are therefore the compacted one and the + /// inline nodes only. + /// The other children value are needed because they can be included into the additional + /// hash, and are required for intermediate hash calculation. + fn encode_compact_proof( + hash_proof_header: Vec, + children: &[Option>], + hash_buf: &mut H::Buffer, + ) -> Vec; + /// Returns branch node encoded for storage, and additional information for hash calculation. /// /// Takes an iterator yielding `ChildReference` and an optional value @@ -156,23 +173,6 @@ pub trait NodeCodecHybrid: NodeCodec { children: impl Iterator>>>, value: Option<&[u8]>, ) -> Vec; - - /// Build compact proof encoding from branch info. - /// - /// - `hash_proof_header`: the part common with the header info from hash. - /// It can be calculated from `branch_node_common` through - /// `ChildProofHeader` call, or directly by `branch_node_for_hash`. - /// - `children`: contains all children, with compact (ommited children) defined as - /// a null length inline node. - /// The children to be include in the proof are therefore the compacted one and the - /// inline nodes only. - /// The other children value are needed because they can be included into the additional - /// hash, and are required for intermediate hash calculation. - fn encode_compact_proof( - hash_proof_header: Vec, - children: &[Option>], - hash_buf: &mut H::Buffer, - ) -> Vec; } /// Information to fetch bytes that needs to be include when calculating a node hash. diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index f0c29808..5cd99677 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -124,7 +124,7 @@ impl<'a, C: NodeCodecHybrid, H: BinaryHasher> StackEntry<'a, C, H> let (node, hybrid) = if !is_inline && hybrid { // TODO factorize with trie_codec let encoded_node = node_data; - C::decode_proof(encoded_node) + C::decode_compact_proof(encoded_node) .map_err(Error::DecodeError)? } else { (C::decode(node_data) diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 3af081fa..06d34ee5 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -486,7 +486,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) for (i, encoded_node) in encoded.iter().enumerate() { let (node, hybrid) = if L::HYBRID_HASH { - L::Codec::decode_proof(encoded_node) + L::Codec::decode_compact_proof(encoded_node) .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))? } else { ( From 123c207e6fffaede7f9d38997126f9f2d952376c Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 17:17:21 +0200 Subject: [PATCH 24/73] reexport, this need refacto, but for quick test in substrate --- trie-db/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index ff9943f4..b2a18765 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -76,7 +76,7 @@ pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybr TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; pub use crate::trie_codec::{decode_compact, encode_compact, binary_additional_hashes}; -pub use ordered_trie::BinaryHasher; +pub use ordered_trie::{HashDBHybrid, HasherHybrid, BinaryHasher}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; From 972a5a3f1f88ec0e484dfe2be084b05a44fdd903 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 18:18:46 +0200 Subject: [PATCH 25/73] moving binaryhasher trait --- hash-db/src/lib.rs | 34 +++++++++++++++++++ ordered-trie/src/lib.rs | 48 +-------------------------- test-support/keccak-hasher/src/lib.rs | 8 ++--- trie-db/src/lib.rs | 4 +-- trie-db/src/node_codec.rs | 2 +- trie-db/src/proof/generate.rs | 3 +- trie-db/src/proof/verify.rs | 4 +-- trie-db/src/trie_codec.rs | 3 +- 8 files changed, 47 insertions(+), 59 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 249ae4eb..0c3e0f1f 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -64,6 +64,40 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } +/// Small trait for to allow using buffer of type [u8; H::LENGTH * 2]. +pub trait BinaryHasher: Hasher { + /// Hash for the empty content (is hash(&[])). + const NULL_HASH: &'static [u8]; + type Buffer: AsRef<[u8]> + AsMut<[u8]> + Default; +} + +/// Test function to use on every binary buffer implementation. +pub fn test_binary_hasher() { + let size = ::LENGTH * 2; + let buf = ::Buffer::default(); + assert_eq!(buf.as_ref().len(), size); + let null_hash = H::hash(&[]); + assert_eq!(H::NULL_HASH, null_hash.as_ref()); +} + +/// A buffer for binary hasher of size 64. +pub struct Buffer64([u8; 64]); +impl AsRef<[u8]> for Buffer64 { + fn as_ref(&self) -> &[u8] { + &self.0[..] + } +} +impl AsMut<[u8]> for Buffer64 { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0[..] + } +} +impl Default for Buffer64 { + fn default() -> Self { + Buffer64([0; 64]) + } +} + /// Trait modelling a plain datastore whose key is a fixed type. /// The caller should ensure that a key only corresponds to /// one value. diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index e2659da0..c60c1227 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -42,7 +42,7 @@ mod rstd { use hash_db::{Prefix, HashDB}; use crate::rstd::vec::Vec; -use hash_db::Hasher; +use hash_db::{Hasher, BinaryHasher}; use crate::rstd::marker::PhantomData; @@ -716,33 +716,6 @@ fn key_node_test() { // test(usize::max_value()); } -/// A buffer for binary hasher of size 64. -pub struct Buffer64([u8; 64]); -impl AsRef<[u8]> for Buffer64 { - fn as_ref(&self) -> &[u8] { - &self.0[..] - } -} -impl AsMut<[u8]> for Buffer64 { - fn as_mut(&mut self) -> &mut [u8] { - &mut self.0[..] - } -} -impl Default for Buffer64 { - fn default() -> Self { - Buffer64([0; 64]) - } -} -/// Test function to use on every binary buffer implementation. -pub fn test_binary_hasher() { - let size = ::LENGTH * 2; - let buf = ::Buffer::default(); - assert_eq!(buf.as_ref().len(), size); - let null_hash = H::hash(&[]); - assert_eq!(H::NULL_HASH, null_hash.as_ref()); - -} - pub trait ProcessNode { /// Callback for an empty trie, return byte representation /// of the hash for the empty trie. @@ -1249,18 +1222,6 @@ mod test { } } - impl BinaryHasher for KeccakHasher { - const NULL_HASH: &'static [u8] = &[197, 210, 70, 1, 134, 247, 35, 60, 146, - 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, - 250, 216, 4, 93, 133, 164, 112]; - type Buffer = Buffer64; - } - - #[test] - fn test_keccack_hasher() { - test_binary_hasher::() - } - fn hashes(l: usize) -> Vec<[u8;32]> { (0..l).map(|i| { let mut hash = ::Out::default(); @@ -1487,13 +1448,6 @@ mod test { } } -/// Small trait for to allow using buffer of type [u8; H::LENGTH * 2]. -pub trait BinaryHasher: Hasher { - /// Hash for the empty content (is hash(&[])). - const NULL_HASH: &'static [u8]; - type Buffer: AsRef<[u8]> + AsMut<[u8]> + Default; -} - pub trait HasherHybrid: BinaryHasher { /// Alternate hash with hybrid proof allowed diff --git a/test-support/keccak-hasher/src/lib.rs b/test-support/keccak-hasher/src/lib.rs index 79a1330c..ea3e3234 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/test-support/keccak-hasher/src/lib.rs @@ -14,7 +14,7 @@ //! Hasher implementation for the Keccak-256 hash -use hash_db::Hasher; +use hash_db::{Hasher, BinaryHasher}; //use hash_db::FixHash; use tiny_keccak::Keccak; use hash256_std_hasher::Hash256StdHasher; @@ -36,16 +36,16 @@ impl Hasher for KeccakHasher { } } -impl ordered_trie::BinaryHasher for KeccakHasher { +impl BinaryHasher for KeccakHasher { const NULL_HASH: &'static [u8] = &[197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112]; - type Buffer = ordered_trie::Buffer64; + type Buffer = hash_db::Buffer64; } #[test] fn test_keccack_hasher() { - ordered_trie::test_binary_hasher::() + hash_db::test_binary_hasher::() } /* TODO this is rather bad trait see if delete?? diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index b2a18765..7d49500f 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -60,7 +60,7 @@ mod nibble; mod node_codec; mod trie_codec; -pub use hash_db::{HashDB, HashDBRef, Hasher}; +pub use hash_db::{HashDB, HashDBRef, Hasher, BinaryHasher}; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::{TrieDBMut, ChildReference}; pub use self::sectriedbmut::SecTrieDBMut; @@ -76,7 +76,7 @@ pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybr TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; pub use crate::trie_codec::{decode_compact, encode_compact, binary_additional_hashes}; -pub use ordered_trie::{HashDBHybrid, HasherHybrid, BinaryHasher}; +pub use ordered_trie::{HashDBHybrid, HasherHybrid}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index e652a374..bb14fbb0 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -18,7 +18,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use ordered_trie::BinaryHasher; +use hash_db::BinaryHasher; use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData}; diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index e042b89d..f22bb22c 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -18,14 +18,13 @@ use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, ops::Range, vec, vec::Vec, }; -use hash_db::Hasher; +use hash_db::{Hasher, BinaryHasher}; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, Result as TrieResult, Trie, TrieError, TrieHash, TrieLayout, NodeCodecHybrid, }; -use ordered_trie::BinaryHasher; struct StackEntry<'a, C: NodeCodec, H> { /// The prefix is the nibble path to the node in the trie. diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index 5cd99677..9bfbeddb 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -19,8 +19,8 @@ use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle}, NodeCodecHybrid, TrieHash, TrieLayout, ChildProofHeader, }; -use hash_db::Hasher; -use ordered_trie::{BinaryHasher, HasherHybrid}; +use hash_db::{Hasher, BinaryHasher}; +use ordered_trie::HasherHybrid; use crate::node_codec::{Bitmap, HashesIter}; diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 06d34ee5..a20d7e2b 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -25,7 +25,8 @@ //! expected to save roughly (n - 1) hashes in size where n is the number of nodes in the partial //! trie. -pub use ordered_trie::{BinaryHasher, HashDBHybrid}; +use ordered_trie::HashDBHybrid; +use hash_db::BinaryHasher; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecHybrid, From 00e5f7bc72bf41bb58c9f5497a445bba26f757a4 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 20:08:55 +0200 Subject: [PATCH 26/73] default trie config function using hybrid trie_iter variants. --- trie-db/src/lib.rs | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 7d49500f..051b9500 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -401,14 +401,20 @@ pub trait TrieLayout { pub trait TrieConfiguration: Sized + TrieLayout { /// Operation to build a trie db from its ordered iterator over its key/values. fn trie_build(db: &mut DB, input: I) -> ::Out where - DB: HashDB, + DB: HashDB + HashDBHybrid, I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - let mut cb = TrieBuilder::new(db); - trie_visit::(input.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + if Self::HYBRID_HASH { + let mut cb = TrieBuilderHybrid::new(db); + trie_visit::(input.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } else { + let mut cb = TrieBuilder::new(db); + trie_visit::(input.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } } /// Determines a trie root given its ordered contents, closed form. fn trie_root(input: I) -> ::Out where @@ -416,9 +422,15 @@ pub trait TrieConfiguration: Sized + TrieLayout { A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - let mut cb = TrieRoot::::default(); - trie_visit::(input.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + if Self::HYBRID_HASH { + let mut cb = TrieRootHybrid::::default(); + trie_visit::(input.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } else { + let mut cb = TrieRoot::::default(); + trie_visit::(input.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } } /// Determines a trie root node's data given its ordered contents, closed form. fn trie_root_unhashed(input: I) -> Vec where @@ -426,9 +438,15 @@ pub trait TrieConfiguration: Sized + TrieLayout { A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - let mut cb = TrieRootUnhashed::::default(); - trie_visit::(input.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + if Self::HYBRID_HASH { + let mut cb = TrieRootUnhashedHybrid::::default(); + trie_visit::(input.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } else { + let mut cb = TrieRootUnhashed::::default(); + trie_visit::(input.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } } /// Encoding of index as a key (when reusing general trie for /// indexed trie). From 4fa78075ba876578e59089b1b17943cccead0459 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 20:27:41 +0200 Subject: [PATCH 27/73] update fuzzer dependencies --- trie-db/fuzz/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index 50ac2501..434d684c 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -10,8 +10,8 @@ cargo-fuzz = true [dependencies] hash-db = { path = "../../hash-db", version = "0.15.2" } -memory-db = { path = "../../memory-db", version = "0.18.1" } -reference-trie = { path = "../../test-support/reference-trie", version = "0.19.0" } +memory-db = { path = "../../memory-db", version = "0.20.0" } +reference-trie = { path = "../../test-support/reference-trie", version = "0.20.0" } keccak-hasher = { path = "../../test-support/keccak-hasher", version = "0.15.2" } [dependencies.trie-db] From 38185b4667ba6ed49397e53c7a2d01ee77e33df7 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 20:32:53 +0200 Subject: [PATCH 28/73] restoring correct number of test in random testing find errors. --- trie-db/src/triedbmut.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 01e340ba..6cf43eaa 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1734,7 +1734,7 @@ mod tests { fn playpen() { env_logger::init(); let mut seed = Default::default(); - for test_i in 0..10 { + for test_i in 0..10000 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); } From 7d5cf1db879496ca1b4079873ac3ca38fb9d9d2f Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Apr 2020 20:39:20 +0200 Subject: [PATCH 29/73] actually no extension works for this ammount, seems related to extension code only. --- trie-db/src/triedbmut.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 6cf43eaa..5b407d26 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1734,7 +1734,7 @@ mod tests { fn playpen() { env_logger::init(); let mut seed = Default::default(); - for test_i in 0..10000 { + for test_i in 0..10 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); } @@ -1777,7 +1777,7 @@ mod tests { // no_extension let mut seed = Default::default(); - for test_i in 0..10 { + for test_i in 0..100000 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); } From ea7ec16a6858dbc8ea0177a96ba065283c952e9d Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 9 Apr 2020 16:04:58 +0200 Subject: [PATCH 30/73] test how to rebuild a proof from encoded nodes only. --- memory-db/src/lib.rs | 22 +++- ordered-trie/src/lib.rs | 18 ++- test-support/reference-trie/src/lib.rs | 71 ++++++++++- trie-db/src/iter_build.rs | 2 +- trie-db/src/lib.rs | 2 +- trie-db/src/node.rs | 9 ++ trie-db/src/node_codec.rs | 155 ++++++++++++++----------- trie-db/src/trie_codec.rs | 2 +- trie-db/src/triedbmut.rs | 6 +- 9 files changed, 204 insertions(+), 83 deletions(-) diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 400e24c1..07e1ed27 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -35,6 +35,7 @@ use std::{ marker::PhantomData, cmp::Eq, borrow::Borrow, + result, }; #[cfg(not(feature = "std"))] @@ -51,6 +52,7 @@ use core::{ cmp::Eq, borrow::Borrow, ops::Range, + result, }; #[cfg(not(feature = "std"))] @@ -608,7 +610,25 @@ where T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, KF: Send + Sync + KeyFunction, { - fn insert_hybrid< + fn insert_hybrid( + &mut self, + prefix: Prefix, + value: &[u8], + callback: fn(&[u8]) -> result::Result, ()>, + ) -> bool { + if let Ok(result) = callback(value) { + if let Some(key) = result { + >::emplace(self, key, prefix, T::from(value)); + } else { + >::insert(self, prefix, value); + } + true + } else { + false + } + } + + fn insert_branch_hybrid< I: Iterator>, I2: Iterator, > ( diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index c60c1227..6392df06 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -1452,6 +1452,7 @@ pub trait HasherHybrid: BinaryHasher { /// Alternate hash with hybrid proof allowed /// TODO expose buffer !! (then memory db use a single buf) + /// TODO EMCH also split depending on proof or not!! fn hash_hybrid< I: Iterator::Out>>, I2: Iterator::Out>, @@ -1517,10 +1518,25 @@ impl HasherHybrid for H { /// Same as HashDB but can modify the value upon storage, and apply /// `HasherHybrid`. pub trait HashDBHybrid: Send + Sync + HashDB { + /// `HashDB` is often use to load content from encoded node. + /// This will not preserve insertion done through `insert_branch_hybrid` calls + /// and break the proof. + /// This function allows to use a callback (usually the call back + /// will check the encoded value with codec and for branch it will + /// emplace over the hash_hybrid key) for changing key of some content. + /// Callback is allow to fail (since it will decode some node this indicates + /// invalid content earlier), in this case we return false. + fn insert_hybrid( + &mut self, + prefix: Prefix, + value: &[u8], + callback: fn(&[u8]) -> rstd::result::Result, ()>, + ) -> bool; + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. - fn insert_hybrid< + fn insert_branch_hybrid< I: Iterator>, I2: Iterator, >( diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index a2e27169..dea56e38 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -87,6 +87,17 @@ pub type RefSecTrieDBMut<'a> = trie_db::SecTrieDBMut<'a, ExtensionLayout>; pub type RefLookup<'a, Q> = trie_db::Lookup<'a, ExtensionLayout, Q>; pub type RefLookupNoExt<'a, Q> = trie_db::Lookup<'a, NoExtensionLayout, Q>; + +/// Typed version of hybrid_hash_node_adapter. +pub fn hybrid_hash_node_adapter_no_ext( + encoded_node: &[u8] +) -> std::result::Result::Out>, ()> { + trie_db::hybrid_hash_node_adapter::< + ReferenceNodeCodecNoExt, + KeccakHasher, + > (encoded_node) +} + pub fn reference_trie_root(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, @@ -399,6 +410,13 @@ enum NodeHeaderNoExt { Leaf(usize), } +impl NodeHeader { + fn is_branch(first_byte: u8) -> bool { + first_byte == BRANCH_NODE_NO_VALUE + || first_byte == BRANCH_NODE_NO_VALUE + } +} + impl Encode for NodeHeader { fn encode_to(&self, output: &mut T) { match self { @@ -491,6 +509,14 @@ impl Decode for NodeHeader { } } +impl NodeHeaderNoExt { + fn is_branch(first_byte: u8) -> bool { + let first_byte = first_byte & (0b11 << 6); + first_byte == BRANCH_WITHOUT_MASK_NO_EXT + || first_byte == BRANCH_WITH_MASK_NO_EXT + } +} + impl Decode for NodeHeaderNoExt { fn decode(input: &mut I) -> Result { let i = input.read_byte()?; @@ -642,8 +668,9 @@ impl<'a> Input for ByteSliceInput<'a> { impl ReferenceNodeCodec { fn decode_plan_internal( data: &[u8], - is_hybrid: bool, + is_proof: bool, ) -> ::std::result::Result<(NodePlan, usize), ::Error> { + let mut result_offset = 0; let mut input = ByteSliceInput::new(data); let node = match NodeHeader::decode(&mut input)? { NodeHeader::Null => NodePlan::Empty, @@ -657,13 +684,14 @@ impl ReferenceNodeCodec { } else { None }; + result_offset = input.offset; let mut children = [ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - if is_hybrid { + if is_proof { children[i] = Some(NodeHandlePlan::Inline(Range { start: 0, end: 0 })); } else { let count = >::decode(&mut input)?.0 as usize; @@ -708,7 +736,7 @@ impl ReferenceNodeCodec { } } }; - Ok((node, input.offset)) + Ok((node, result_offset)) } } @@ -829,6 +857,21 @@ impl NodeCodecHybrid for ReferenceNodeCodec { ) -> Vec { encode_proof_internal::(hash_proof_header, children, hash_buf) } + + fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)> { + if data.len() > 0 { + if NodeHeader::is_branch(data[0]) { + if let Ok((node, offset)) = Self::decode_plan_internal(data, false) { + let header = ChildProofHeader::Range( Range { + start: 0, + end: offset, + }); + return Some((node, header)) + } + } + } + None + } } impl ReferenceNodeCodec { @@ -996,8 +1039,9 @@ fn encode_proof_internal( impl ReferenceNodeCodecNoExt { fn decode_plan_internal( data: &[u8], - is_hybrid: bool, + is_proof: bool, ) -> ::std::result::Result<(NodePlan, usize), ::Error> { + let mut result_offset = 0; let mut input = ByteSliceInput::new(data); let node = match NodeHeaderNoExt::decode(&mut input)? { NodeHeaderNoExt::Null => NodePlan::Empty, @@ -1023,9 +1067,10 @@ impl ReferenceNodeCodecNoExt { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + result_offset = input.offset; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - if is_hybrid { + if is_proof { children[i] = Some(NodeHandlePlan::Inline(Range { start: 0, end: 0 })); } else { let count = >::decode(&mut input)?.0 as usize; @@ -1062,7 +1107,7 @@ impl ReferenceNodeCodecNoExt { } } }; - Ok((node, input.offset)) + Ok((node, result_offset)) } } @@ -1269,6 +1314,20 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { encode_proof_internal::(hash_proof_header, children, hash_buf) } + fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)> { + if data.len() > 0 { + if NodeHeaderNoExt::is_branch(data[0]) { + if let Ok((node, offset)) = Self::decode_plan_internal(data, false) { + let header = ChildProofHeader::Range( Range { + start: 0, + end: offset, + }); + return Some((node, header)) + } + } + } + None + } } /// Compare trie builder and in memory trie. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 70f91780..0271777a 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -458,7 +458,7 @@ impl<'a, H: HasherHybrid, V, DB: HashDBHybrid> ProcessEncodedNode< Some(Some(v.clone())), None => None, }); - self.db.insert_hybrid( + self.db.insert_branch_hybrid( prefix, &encoded_node[..], common.header(&encoded_node[..]), diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 051b9500..e1cb3459 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -71,7 +71,7 @@ pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; pub use crate::node_codec::{NodeCodec, NodeCodecHybrid, Partial, HashDBHybridDyn, ChildProofHeader, - Bitmap, BITMAP_LENGTH, HashesPlan}; + Bitmap, BITMAP_LENGTH, HashesPlan, hybrid_hash_node_adapter}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybrid, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index d75af745..46be0038 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -85,6 +85,15 @@ impl NodeHandlePlan { NodeHandlePlan::Inline(range) => NodeHandle::Inline(&data[range.clone()]), } } + /// Range of node handle definition in encoded data. + /// TODO EMCH this breaks design a bit, maybe return H::Out + /// or put a util function. + pub fn range<'a, 'b>(&'a self) -> Range { + match self { + NodeHandlePlan::Hash(range) => range.clone(), + NodeHandlePlan::Inline(range) => range.clone(), + } + } } /// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice. The diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index bb14fbb0..64e74354 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -20,7 +20,7 @@ use crate::node::{Node, NodePlan}; use crate::ChildReference; use hash_db::BinaryHasher; -use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range, marker::PhantomData}; +use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range}; /// Representation of a nible slice (right aligned). /// It contains a right aligned padded first byte (first pair element is the number of nibbles @@ -137,6 +137,12 @@ pub trait NodeCodecHybrid: NodeCodec { hash_buf: &mut H::Buffer, ) -> Vec; + /// Does the encoded content need a hybrid proof. + /// With current hybrid proof capability this is strictly + /// the same as checking if the node is a branch. + /// It return the proof header and the NodePlan if hybrid proof is needed. + fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)>; + /// Returns branch node encoded for storage, and additional information for hash calculation. /// /// Takes an iterator yielding `ChildReference` and an optional value @@ -207,7 +213,7 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { /// /// TODO warn semantic of children differs from HashDBHybrid (in HashDBHybrid it is the /// children of the binary hasher, here it is the children of the patricia merkle trie). - fn insert_hybrid( + fn insert_branch_hybrid( &mut self, prefix: Prefix, value: &[u8], @@ -217,7 +223,7 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { } impl> HashDBHybridDyn for C { - fn insert_hybrid( + fn insert_branch_hybrid( &mut self, prefix: Prefix, value: &[u8], @@ -225,14 +231,15 @@ impl> HashDBHybridDyn for C { common: ChildProofHeader, ) -> H::Out { - // TODO factor this with iter_build (just use the trait) + // TODO factor this with iter_build (just use the trait) also use in adapter from codec let nb_children = children.iter().filter(|v| v.is_some()).count(); - let children = HybridLayoutIterValues::new( - children.iter().filter_map(|v| v.as_ref()), - value, - ); + let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { + let mut dest = H::Out::default(); + dest.as_mut()[..range.len()].copy_from_slice(&value[range.clone()]); + dest + })); - >::insert_hybrid( + >::insert_branch_hybrid( self, prefix, value, @@ -265,66 +272,6 @@ impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDBHybridDyn { } } -// TODO this using a buffer is bad (we should switch -// binary hasher to use slice as input (or be able to)) -pub struct HybridLayoutIterValues<'a, HO, I> { - children: I, - node: &'a [u8], - _ph: PhantomData, -} -/* -code snippet for children iter: -HybridLayoutIterValues::new(nb_children, children, value) - .map(|(is_defined, v)| { - debug_assert!(is_defined); - v - }); -code snippet for proof - let iter = HybridLayoutIterValues::new(nb_children, children, value) - .zip(iter_key) - .filter_map(|((is_defined, hash), key)| if is_defined { - Some((key, hash)) - } else { - None - }); -*/ - -impl<'a, HO: Default, I> HybridLayoutIterValues<'a, HO, I> { - pub fn new(children: I, node: &'a[u8]) -> Self { - HybridLayoutIterValues { - children, - node, - _ph: PhantomData, - } - } -} - -impl<'a, HO: AsMut<[u8]> + Default, I: Iterator>> Iterator for HybridLayoutIterValues<'a, HO, I> { - type Item = Option; - - fn next(&mut self) -> Option { - if let Some(range) = self.children.next() { - let range_len = range.len(); - if range_len == 0 { - // this is for undefined proof hash - return None; - } - let mut dest = HO::default(); - dest.as_mut()[..range_len].copy_from_slice(&self.node[range.clone()]); - /* inherent to default?? TODO consider doing it, but if refacto - * this will be part of hasher (when run with slice as input) - * for i in range_len..dest.len() { - dest[i] = 0; - }*/ - // TODO the input iterator is HO but could really be &HO, would need some - // change on trie_root to. - Some(Some(dest)) - } else { - None - } - } -} - /// Children bitmap codec for radix 16 trie. pub struct Bitmap(u16); @@ -429,3 +376,73 @@ impl Iterator for HashesPlan { (size, Some(size)) } } + +/// Adapter standard implementation to use with `HashDBInsertComplex` function. +pub fn hybrid_hash_node_adapter, Hasher: HasherHybrid>( + encoded_node: &[u8] +) -> crate::rstd::result::Result, ()> { + Ok(if let Some((node, common)) = Codec::need_hybrid_proof(encoded_node) { + match node { + NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { + let range = range.range(); + let mut dest = Hasher::Out::default(); + dest.as_mut()[..range.len()].copy_from_slice(&encoded_node[range]); + dest + })); + Some(Hasher::hash_hybrid( + common.header(encoded_node), + nb_children, + children, + EmptyIter::default(), + false, // not a proof, will not fail + ).expect("not a proof, does not fail")) // TODO EMCH split function in two variants! + }, + _ => unreachable!("hybrid only touch branch node"), + } + } else { + None + }) +} + +#[test] +fn test_hybrid_hash_node_adapter() { + use reference_trie::{RefTrieDBMutNoExt, TrieMut, hybrid_hash_node_adapter_no_ext}; + use crate::DBValue; + use memory_db::{MemoryDB, HashKey}; + use keccak_hasher::KeccakHasher; + use hash_db::EMPTY_PREFIX; + + + let mut db = MemoryDB::, DBValue>::default(); + let data = vec![ + (vec![0], vec![251; 34]), + (vec![0, 1], vec![252; 34]), + (vec![0, 1, 2], vec![255; 32]), + (vec![0, 1], vec![0; 38]), + ]; + let mut root = Default::default(); + { + let mut t = RefTrieDBMutNoExt::new(&mut db, &mut root); + for i in 0..data.len() { + let key: &[u8]= &data[i].0; + let value: &[u8] = &data[i].1; + t.insert(key, value).unwrap(); + } + } + let mut db2 = MemoryDB::, DBValue>::default(); + for (_key, (value, rc)) in db.clone().drain() { + if rc > 0 { + if !db2.insert_hybrid( + EMPTY_PREFIX, + &value[..], + hybrid_hash_node_adapter_no_ext, + ) { + panic!("invalid encoded node in proof"); + } + } + } + + assert!(db == db2); +} diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index a20d7e2b..de7cd792 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -545,7 +545,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) None } }); - db.insert_hybrid( + db.insert_branch_hybrid( prefix.as_prefix(), &node_data[..], common.header(&node_data[..]), diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 5b407d26..6ec17fc5 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1473,7 +1473,7 @@ where #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:#x?}", &encoded_root[..]); if let Some(children) = register_children { - *self.root = self.db.insert_hybrid( + *self.root = self.db.insert_branch_hybrid( EMPTY_PREFIX, &encoded_root[..], &children[..], @@ -1531,7 +1531,7 @@ where }; if encoded.len() >= L::Hash::LENGTH { let hash = if let Some(children) = register_children { - self.db.insert_hybrid( + self.db.insert_branch_hybrid( prefix.as_prefix(), &encoded[..], &children[..], @@ -1777,7 +1777,7 @@ mod tests { // no_extension let mut seed = Default::default(); - for test_i in 0..100000 { + for test_i in 0..100 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); } From e2dd371f66eea80a4df1aad21c0f3f254ed2f75f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 12 May 2020 11:20:39 +0200 Subject: [PATCH 31/73] Move trait from ordered trie to hash db, now the hybrid implementation is defined by the trie layout --- hash-db/src/lib.rs | 53 ++++++++ memory-db/src/lib.rs | 3 +- ordered-trie/src/lib.rs | 176 ++++++++++--------------- test-support/reference-trie/src/lib.rs | 38 +++--- trie-db/fuzz/src/lib.rs | 2 +- trie-db/src/fatdb.rs | 3 +- trie-db/src/fatdbmut.rs | 3 +- trie-db/src/iter_build.rs | 7 +- trie-db/src/iterator.rs | 3 +- trie-db/src/lib.rs | 5 +- trie-db/src/node_codec.rs | 12 +- trie-db/src/proof/generate.rs | 12 +- trie-db/src/proof/verify.rs | 6 +- trie-db/src/recorder.rs | 3 +- trie-db/src/sectriedb.rs | 3 +- trie-db/src/sectriedbmut.rs | 3 +- trie-db/src/trie_codec.rs | 9 +- trie-db/src/triedb.rs | 6 +- trie-db/src/triedbmut.rs | 6 +- 19 files changed, 180 insertions(+), 173 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 0c3e0f1f..3041bb0f 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -219,3 +219,56 @@ impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { fn as_plain_db(&self) -> &dyn PlainDB { &**self } fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn PlainDB + 'b) { &mut **self } } + +/// Same as HashDB but can modify the value upon storage, and apply +/// `HasherHybrid`. +pub trait HashDBHybrid: Send + Sync + HashDB { + /// `HashDB` is often use to load content from encoded node. + /// This will not preserve insertion done through `insert_branch_hybrid` calls + /// and break the proof. + /// This function allows to use a callback (usually the call back + /// will check the encoded value with codec and for branch it will + /// emplace over the hash_hybrid key) for changing key of some content. + /// Callback is allow to fail (since it will decode some node this indicates + /// invalid content earlier), in this case we return false. + fn insert_hybrid( + &mut self, + prefix: Prefix, + value: &[u8], + callback: fn(&[u8]) -> core::result::Result, ()>, + ) -> bool; + + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions + /// are counted and the equivalent number of `remove()`s must be performed before the data + /// is considered dead. + fn insert_branch_hybrid< + I: Iterator>, + I2: Iterator, + >( + &mut self, + prefix: Prefix, + value: &[u8], + no_child_value: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> H::Out; +} + +pub trait HasherHybrid: BinaryHasher { + + /// Alternate hash with hybrid proof allowed + /// TODO expose buffer !! (then memory db use a single buf) + /// TODO EMCH also split depending on proof or not!! + fn hash_hybrid< + I: Iterator::Out>>, + I2: Iterator::Out>, + >( + x: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> Option; +} diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 07e1ed27..ad61b52b 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -19,9 +19,8 @@ #[cfg(not(feature = "std"))] extern crate alloc; -use ordered_trie::{HashDBHybrid, HasherHybrid}; use hash_db::{HashDB, HashDBRef, PlainDB, PlainDBRef, Hasher as KeyHasher, - AsHashDB, AsPlainDB, Prefix}; + AsHashDB, AsPlainDB, Prefix, HashDBHybrid, HasherHybrid}; use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; #[cfg(feature = "deprecated")] #[cfg(feature = "std")] diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 6392df06..8b09a121 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -39,10 +39,9 @@ mod rstd { } -use hash_db::{Prefix, HashDB}; use crate::rstd::vec::Vec; -use hash_db::{Hasher, BinaryHasher}; +use hash_db::{Hasher, HasherHybrid, BinaryHasher}; use crate::rstd::marker::PhantomData; @@ -1129,6 +1128,76 @@ pub fn trie_root_from_proof( } } +#[repr(transparent)] +/// Hasher hybrid using an ordered trie +pub struct OrderedTrieHasher(H); + +impl HasherHybrid for OrderedTrieHasher { + fn hash_hybrid< + I: Iterator::Out>>, + I2: Iterator::Out>, + >( + header: &[u8], + nb_children: usize, + children: I, + additional_hashes: I2, + proof: bool, + ) -> Option { + let seq_trie = SequenceBinaryTree::new(0, 0, nb_children); + + let mut hash_buf2 = ::Buffer::default(); + let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); + let hash = if !proof { + // full node + let iter = children.filter_map(|v| v); // TODO assert same number as count + crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof) + } else { + // proof node, UsizeKeyNode should be big enough for hasher hybrid + // case. + let iter_key = seq_trie.iter_path_node_key::(None); + let iter = children + .zip(iter_key) + .filter_map(|(value, key)| if let Some(value) = value { + Some((key, value)) + } else { + None + }); + if let Some(hash) = crate::trie_root_from_proof( + &seq_trie, + iter, + additional_hashes, + &mut callback_read_proof, + false, + ) { + hash + } else { + return None; + } + }; + // TODO really need a real hash trait + let mut buf = Vec::with_capacity(header.len() + hash.as_ref().len()); + buf.extend_from_slice(header); + buf.extend_from_slice(hash.as_ref()); + Some(H::hash(buf.as_slice())) + } +} + +impl BinaryHasher for OrderedTrieHasher { + const NULL_HASH: &'static [u8] = ::NULL_HASH; + type Buffer = ::Buffer; +} + +impl Hasher for OrderedTrieHasher { + type Out = ::Out; + type StdHasher = ::StdHasher; + const LENGTH: usize = ::LENGTH; + + #[inline] + fn hash(x: &[u8]) -> Self::Out { + ::hash(x) + } +} + #[cfg(test)] mod test { use keccak_hasher::KeccakHasher; @@ -1447,106 +1516,3 @@ mod test { } } } - -pub trait HasherHybrid: BinaryHasher { - - /// Alternate hash with hybrid proof allowed - /// TODO expose buffer !! (then memory db use a single buf) - /// TODO EMCH also split depending on proof or not!! - fn hash_hybrid< - I: Iterator::Out>>, - I2: Iterator::Out>, - >( - x: &[u8], - nb_children: usize, - children: I, - additional_hashes: I2, - proof: bool, - ) -> Option; -} - -impl HasherHybrid for H { - fn hash_hybrid< - I: Iterator::Out>>, - I2: Iterator::Out>, - >( - header: &[u8], - nb_children: usize, - children: I, - additional_hashes: I2, - proof: bool, - ) -> Option { - let seq_trie = SequenceBinaryTree::new(0, 0, nb_children); - - let mut hash_buf2 = ::Buffer::default(); - let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); - let hash = if !proof { - // full node - let iter = children.filter_map(|v| v); // TODO assert same number as count - crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof) - } else { - // proof node, UsizeKeyNode should be big enough for hasher hybrid - // case. - let iter_key = seq_trie.iter_path_node_key::(None); - let iter = children - .zip(iter_key) - .filter_map(|(value, key)| if let Some(value) = value { - Some((key, value)) - } else { - None - }); - if let Some(hash) = crate::trie_root_from_proof( - &seq_trie, - iter, - additional_hashes, - &mut callback_read_proof, - false, - ) { - hash - } else { - return None; - } - }; - // TODO really need a real hash trait - let mut buf = Vec::with_capacity(header.len() + hash.as_ref().len()); - buf.extend_from_slice(header); - buf.extend_from_slice(hash.as_ref()); - Some(H::hash(buf.as_slice())) - } -} - -/// Same as HashDB but can modify the value upon storage, and apply -/// `HasherHybrid`. -pub trait HashDBHybrid: Send + Sync + HashDB { - /// `HashDB` is often use to load content from encoded node. - /// This will not preserve insertion done through `insert_branch_hybrid` calls - /// and break the proof. - /// This function allows to use a callback (usually the call back - /// will check the encoded value with codec and for branch it will - /// emplace over the hash_hybrid key) for changing key of some content. - /// Callback is allow to fail (since it will decode some node this indicates - /// invalid content earlier), in this case we return false. - fn insert_hybrid( - &mut self, - prefix: Prefix, - value: &[u8], - callback: fn(&[u8]) -> rstd::result::Result, ()>, - ) -> bool; - - /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions - /// are counted and the equivalent number of `remove()`s must be performed before the data - /// is considered dead. - fn insert_branch_hybrid< - I: Iterator>, - I2: Iterator, - >( - &mut self, - prefix: Prefix, - value: &[u8], - no_child_value: &[u8], - nb_children: usize, - children: I, - additional_hashes: I2, - proof: bool, - ) -> H::Out; -} diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index dea56e38..86785bec 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -29,16 +29,15 @@ use trie_db::{ TrieBuilderHybrid, TrieRootHybrid, Partial, - BinaryHasher, + HasherHybrid, ChildProofHeader, HashesPlan, binary_additional_hashes, }; use std::borrow::Borrow; -use keccak_hasher::KeccakHasher; pub use trie_db::{ - decode_compact, encode_compact, HashDBHybridDyn, + decode_compact, encode_compact, HashDBHybrid, HashDBHybridDyn, nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, NodeCodecHybrid, Trie, TrieConfiguration, TrieDB, TrieDBIterator, TrieDBMut, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, TrieMut, Bitmap, BITMAP_LENGTH, @@ -48,6 +47,9 @@ pub mod node { pub use trie_db::node::Node; } +/// Running reference on keccak with hybrid ordered trie implementation. +pub type KeccakHasher = ordered_trie::OrderedTrieHasher; + /// Trie layout using extension nodes. pub struct ExtensionLayout; @@ -64,17 +66,17 @@ impl TrieConfiguration for ExtensionLayout { } /// generic hasher. pub struct GenericNoExtensionLayout(PhantomData); -impl TrieLayout for GenericNoExtensionLayout { +impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; const HYBRID_HASH: bool = true; type Hash = H; type Codec = ReferenceNodeCodecNoExt; } -impl TrieConfiguration for GenericNoExtensionLayout { } +impl TrieConfiguration for GenericNoExtensionLayout { } /// Trie layout without extension nodes. -pub type NoExtensionLayout = GenericNoExtensionLayout; +pub type NoExtensionLayout = GenericNoExtensionLayout; pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; pub type RefTrieDBNoExt<'a> = trie_db::TrieDB<'a, NoExtensionLayout>; @@ -850,7 +852,7 @@ impl NodeCodecHybrid for ReferenceNodeCodec { unreachable!() } - fn encode_compact_proof( + fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], hash_buf: &mut BH::Buffer, @@ -996,7 +998,7 @@ fn decode_plan_compact_proof_internal( Ok((node, hashes_plan)) } -fn encode_proof_internal( +fn encode_proof_internal( mut result: Vec, children: &[Option>], hash_buf: &mut H::Buffer, @@ -1306,7 +1308,7 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { ).0 } - fn encode_compact_proof( + fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], hash_buf: &mut BH::Buffer, @@ -1331,7 +1333,7 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { } /// Compare trie builder and in memory trie. -pub fn compare_implementations + ordered_trie::HashDBHybrid + Eq> ( +pub fn compare_implementations + HashDBHybrid + Eq> ( data: Vec<(Vec, Vec)>, mut memdb: X, mut hashdb: X, @@ -1377,7 +1379,7 @@ pub fn compare_implementations + orde /// Compare trie builder and trie root implementations. pub fn compare_root( data: Vec<(Vec, Vec)>, - mut memdb: impl ordered_trie::HashDBHybrid, + mut memdb: impl HashDBHybrid, ) { let root_new = { let mut cb = TrieRootHybrid::::default(); @@ -1463,7 +1465,7 @@ pub fn calc_root_build( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + ordered_trie::HashDBHybrid, + DB: hash_db::HashDB + HashDBHybrid, { let mut cb = TrieBuilderHybrid::new(hashdb); trie_visit::(data.into_iter(), &mut cb); @@ -1480,7 +1482,7 @@ pub fn calc_root_build_no_extension( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + ordered_trie::HashDBHybrid, + DB: hash_db::HashDB + HashDBHybrid, { let mut cb = TrieBuilderHybrid::new(hashdb); trie_db::trie_visit::(data.into_iter(), &mut cb); @@ -1491,8 +1493,8 @@ pub fn calc_root_build_no_extension( /// This uses the variant without extension nodes. pub fn compare_implementations_no_extension( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, - mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, + mut memdb: impl hash_db::HashDB + HashDBHybrid, + mut hashdb: impl hash_db::HashDB + HashDBHybrid, ) { let root_new = { let mut cb = TrieBuilderHybrid::new(&mut hashdb); @@ -1534,8 +1536,8 @@ pub fn compare_implementations_no_extension( /// ordering before running when trie_build expect correct ordering). pub fn compare_implementations_no_extension_unordered( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, - mut hashdb: impl hash_db::HashDB + ordered_trie::HashDBHybrid, + mut memdb: impl hash_db::HashDB + HashDBHybrid, + mut hashdb: impl hash_db::HashDB + HashDBHybrid, ) { let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { @@ -1579,7 +1581,7 @@ pub fn compare_implementations_no_extension_unordered( /// its input test data. pub fn compare_no_extension_insert_remove( data: Vec<(bool, Vec, Vec)>, - mut memdb: impl ordered_trie::HashDBHybrid, + mut memdb: impl HashDBHybrid, ) { let mut data2 = std::collections::BTreeMap::new(); let mut root = Default::default(); diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 3f36debf..9261453d 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -14,7 +14,6 @@ use hash_db::Hasher; -use keccak_hasher::KeccakHasher; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ calc_root_no_extension, @@ -27,6 +26,7 @@ use reference_trie::{ RefTrieDBMutNoExt, RefTrieDBNoExt, TrieDBIterator, + KeccakHasher, }; use std::convert::TryInto; use trie_db::{DBValue, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut}; diff --git a/trie-db/src/fatdb.rs b/trie-db/src/fatdb.rs index b3f43138..f6e0cd57 100644 --- a/trie-db/src/fatdb.rs +++ b/trie-db/src/fatdb.rs @@ -130,8 +130,7 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use crate::DBValue; - use keccak_hasher::KeccakHasher; - use reference_trie::{RefFatDBMut, RefFatDB, Trie, TrieMut}; + use reference_trie::{RefFatDBMut, RefFatDB, Trie, TrieMut, KeccakHasher}; #[test] fn fatdb_to_trie() { diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs index 7a53177a..eb41da1c 100644 --- a/trie-db/src/fatdbmut.rs +++ b/trie-db/src/fatdbmut.rs @@ -112,8 +112,7 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use hash_db::{Hasher, EMPTY_PREFIX}; - use keccak_hasher::KeccakHasher; - use reference_trie::{RefFatDBMut, RefTrieDB, Trie, TrieMut}; + use reference_trie::{RefFatDBMut, RefTrieDB, Trie, TrieMut, KeccakHasher}; #[test] fn fatdbmut_to_trie() { diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 0271777a..edc4b7b9 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -17,8 +17,7 @@ //! implementation. //! See `trie_visit` function. -use hash_db::{Hasher, HashDB, Prefix}; -use ordered_trie::{HashDBHybrid, HasherHybrid}; +use hash_db::{Hasher, HashDB, Prefix, HasherHybrid, HashDBHybrid}; use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range}; use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; @@ -708,7 +707,9 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashe mod test { use crate::DBValue; use memory_db::{MemoryDB, HashKey, PrefixedKey}; - use keccak_hasher::KeccakHasher; + + type KeccakHasher = ordered_trie::OrderedTrieHasher; + #[test] fn trie_root_empty () { diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 9fe56041..a0d350fa 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -388,11 +388,10 @@ mod tests { use crate::DBValue; use hex_literal::hex; use hash_db::{HashDB, Hasher}; - use keccak_hasher::KeccakHasher; use reference_trie::{ RefTrieDB, RefTrieDBMut, TrieError, TrieMut, TrieIterator, TrieDBNodeIterator, NibbleSlice, NibbleVec, - node::Node, + node::Node, KeccakHasher, }; use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt}; diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index e1cb3459..a468873f 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -60,7 +60,7 @@ mod nibble; mod node_codec; mod trie_codec; -pub use hash_db::{HashDB, HashDBRef, Hasher, BinaryHasher}; +pub use hash_db::{HashDB, HashDBRef, Hasher, BinaryHasher, HasherHybrid, HashDBHybrid}; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::{TrieDBMut, ChildReference}; pub use self::sectriedbmut::SecTrieDBMut; @@ -76,7 +76,6 @@ pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybr TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; pub use crate::trie_codec::{decode_compact, encode_compact, binary_additional_hashes}; -pub use ordered_trie::{HashDBHybrid, HasherHybrid}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; @@ -390,7 +389,7 @@ pub trait TrieLayout { /// and dummy implementation can be used. const HYBRID_HASH: bool; /// Hasher to use for this trie. - type Hash: BinaryHasher; + type Hash: HasherHybrid; /// Codec to use (needs to match hasher and nibble ops). type Codec: NodeCodecHybrid::Out>; } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 64e74354..99356614 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -18,7 +18,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use hash_db::BinaryHasher; +use hash_db::HasherHybrid; use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range}; @@ -131,7 +131,7 @@ pub trait NodeCodecHybrid: NodeCodec { /// inline nodes only. /// The other children value are needed because they can be included into the additional /// hash, and are required for intermediate hash calculation. - fn encode_compact_proof( + fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], hash_buf: &mut H::Buffer, @@ -203,8 +203,7 @@ impl ChildProofHeader { } } -use ordered_trie::{HashDBHybrid, HasherHybrid}; -use hash_db::{HashDB, Prefix, HashDBRef, Hasher}; +use hash_db::{HashDB, Prefix, HashDBRef, Hasher, HashDBHybrid}; pub trait HashDBHybridDyn: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions @@ -391,7 +390,7 @@ pub fn hybrid_hash_node_adapter, H dest.as_mut()[..range.len()].copy_from_slice(&encoded_node[range]); dest })); - Some(Hasher::hash_hybrid( + Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( common.header(encoded_node), nb_children, children, @@ -408,10 +407,9 @@ pub fn hybrid_hash_node_adapter, H #[test] fn test_hybrid_hash_node_adapter() { - use reference_trie::{RefTrieDBMutNoExt, TrieMut, hybrid_hash_node_adapter_no_ext}; + use reference_trie::{RefTrieDBMutNoExt, TrieMut, hybrid_hash_node_adapter_no_ext, KeccakHasher}; use crate::DBValue; use memory_db::{MemoryDB, HashKey}; - use keccak_hasher::KeccakHasher; use hash_db::EMPTY_PREFIX; diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index f22bb22c..8525b331 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -18,7 +18,7 @@ use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, ops::Range, vec, vec::Vec, }; -use hash_db::{Hasher, BinaryHasher}; +use hash_db::{Hasher, HasherHybrid}; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, @@ -45,9 +45,9 @@ struct StackEntry<'a, C: NodeCodec, H> { _marker: PhantomData<(C, H)>, } -impl<'a, C: NodeCodecHybrid, H: BinaryHasher> StackEntry<'a, C, H> +impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> where - H: BinaryHasher, + H: HasherHybrid, { fn new( prefix: LeftNibbleSlice<'a>, @@ -268,7 +268,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) .collect::>(); keys.sort(); keys.dedup(); - let mut hash_buf = ::Buffer::default(); + let mut hash_buf = ::Buffer::default(); let hash_buf = &mut hash_buf; @@ -537,7 +537,7 @@ fn value_with_omission<'a>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( +fn unwind_stack( stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, @@ -545,7 +545,7 @@ fn unwind_stack( hash_buf: &mut H::Buffer, ) -> TrieResult<(), C::HashOut, C::Error> where - H: BinaryHasher, + H: HasherHybrid, { while let Some(entry) = stack.pop() { match maybe_key { diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index 9bfbeddb..15c63425 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -19,9 +19,9 @@ use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle}, NodeCodecHybrid, TrieHash, TrieLayout, ChildProofHeader, }; -use hash_db::{Hasher, BinaryHasher}; -use ordered_trie::HasherHybrid; +use hash_db::{Hasher, BinaryHasher, HasherHybrid}; use crate::node_codec::{Bitmap, HashesIter}; +use ordered_trie::OrderedTrieHasher; /// Errors that may occur during proof verification. Most of the errors types simply indicate that @@ -503,7 +503,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } }); - if let Some(h) = L::Hash::hash_hybrid( + if let Some(h) = OrderedTrieHasher::::hash_hybrid( &common.header(node_data.as_slice())[..], nb_children, children, diff --git a/trie-db/src/recorder.rs b/trie-db/src/recorder.rs index 68d571cf..ea4c18fd 100644 --- a/trie-db/src/recorder.rs +++ b/trie-db/src/recorder.rs @@ -79,8 +79,7 @@ impl Recorder { mod tests { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use keccak_hasher::KeccakHasher; - use reference_trie::{RefTrieDB, RefTrieDBMut, Trie, TrieMut, Recorder, Record}; + use reference_trie::{RefTrieDB, RefTrieDBMut, Trie, TrieMut, Recorder, Record, KeccakHasher}; #[test] fn basic_recorder() { diff --git a/trie-db/src/sectriedb.rs b/trie-db/src/sectriedb.rs index a050290c..82d7160f 100644 --- a/trie-db/src/sectriedb.rs +++ b/trie-db/src/sectriedb.rs @@ -87,8 +87,7 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use keccak_hasher::KeccakHasher; - use reference_trie::{RefTrieDBMut, RefSecTrieDB, Trie, TrieMut}; + use reference_trie::{RefTrieDBMut, RefSecTrieDB, Trie, TrieMut, KeccakHasher}; use crate::DBValue; #[test] diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs index 0e94134c..00596822 100644 --- a/trie-db/src/sectriedbmut.rs +++ b/trie-db/src/sectriedbmut.rs @@ -93,8 +93,7 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use keccak_hasher::KeccakHasher; - use reference_trie::{RefTrieDB, RefSecTrieDBMut, Trie, TrieMut}; + use reference_trie::{RefTrieDB, RefSecTrieDBMut, Trie, TrieMut, KeccakHasher}; use crate::DBValue; #[test] diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index de7cd792..85d9b2ae 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -25,8 +25,7 @@ //! expected to save roughly (n - 1) hashes in size where n is the number of nodes in the partial //! trie. -use ordered_trie::HashDBHybrid; -use hash_db::BinaryHasher; +use hash_db::{HasherHybrid, HashDBHybrid}; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecHybrid, @@ -104,7 +103,7 @@ impl EncoderStackEntry { hash_buf: &mut H::Buffer, ) -> Result, C::HashOut, C::Error> where - H: BinaryHasher, + H: HasherHybrid, { let node_data = self.node.data(); Ok(match self.node.node_plan() { @@ -210,7 +209,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE let mut output = Vec::new(); // TODO make it optional and replace boolean is_hybrid - let mut hash_buf = ::Buffer::default(); + let mut hash_buf = ::Buffer::default(); let hash_buf = &mut hash_buf; // The stack of nodes through a path in the trie. Each entry is a child node of the preceding @@ -579,7 +578,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) /// indicates if it is included in the proof (inline node or /// compacted node). /// - `hash_buf` a buffer of the right size to compute the hash. -pub fn binary_additional_hashes( +pub fn binary_additional_hashes( children: &[Option>], in_proof_children: &[bool], hash_buf: &mut H::Buffer, diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 71d27f32..b682164d 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -35,9 +35,8 @@ use crate::rstd::{fmt, vec::Vec}; /// # Example /// ``` /// use hash_db::Hasher; -/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut}; +/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut, KeccakHasher}; /// use trie_db::DBValue; -/// use keccak_hasher::KeccakHasher; /// use memory_db::*; /// /// let mut memdb = MemoryDB::, _>::default(); @@ -332,10 +331,9 @@ impl<'a, L: TrieLayout> Iterator for TrieDBIterator<'a, L> { #[cfg(test)] mod tests { use memory_db::{MemoryDB, PrefixedKey}; - use keccak_hasher::KeccakHasher; use crate::DBValue; use reference_trie::{RefTrieDB, RefTrieDBMut, RefLookup, Trie, TrieMut, NibbleSlice}; - use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt}; + use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt, KeccakHasher}; use hex_literal::hex; #[test] diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 6ec17fc5..a90345b2 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -422,9 +422,8 @@ impl<'a, H> Index<&'a StorageHandle> for NodeStorage { /// # Example /// ``` /// use hash_db::Hasher; -/// use reference_trie::{RefTrieDBMut, TrieMut}; +/// use reference_trie::{RefTrieDBMut, TrieMut, KeccakHasher}; /// use trie_db::DBValue; -/// use keccak_hasher::KeccakHasher; /// use memory_db::*; /// /// let mut memdb = MemoryDB::, DBValue>::default(); @@ -1678,9 +1677,8 @@ mod tests { use crate::DBValue; use memory_db::{MemoryDB, PrefixedKey}; use hash_db::Hasher; - use keccak_hasher::KeccakHasher; use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, - ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, + ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, KeccakHasher, reference_trie_root_no_extension_iter_build as reference_trie_root_no_extension}; use crate::nibble::BackingByteVec; From 8d8cb3a37586323ba434842a9bcee669769fff20 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 12 May 2020 11:27:43 +0200 Subject: [PATCH 32/73] rename reference keccak hash to RefHasher --- test-support/reference-trie/src/lib.rs | 84 +++++++++++++------------- trie-db/fuzz/src/lib.rs | 4 +- trie-db/src/fatdb.rs | 4 +- trie-db/src/fatdbmut.rs | 12 ++-- trie-db/src/iter_build.rs | 18 +++--- trie-db/src/iterator.rs | 8 +-- trie-db/src/node_codec.rs | 6 +- trie-db/src/recorder.rs | 10 +-- trie-db/src/sectriedb.rs | 6 +- trie-db/src/sectriedbmut.rs | 6 +- trie-db/src/triedb.rs | 24 ++++---- trie-db/src/triedbmut.rs | 62 +++++++++---------- 12 files changed, 121 insertions(+), 123 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 86785bec..a89cf741 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -47,8 +47,8 @@ pub mod node { pub use trie_db::node::Node; } -/// Running reference on keccak with hybrid ordered trie implementation. -pub type KeccakHasher = ordered_trie::OrderedTrieHasher; +/// Reference hasher is a keccak hasher with hybrid ordered trie implementation. +pub type RefHasher = ordered_trie::OrderedTrieHasher; /// Trie layout using extension nodes. pub struct ExtensionLayout; @@ -56,8 +56,8 @@ pub struct ExtensionLayout; impl TrieLayout for ExtensionLayout { const USE_EXTENSION: bool = true; const HYBRID_HASH: bool = true; - type Hash = KeccakHasher; - type Codec = ReferenceNodeCodec; + type Hash = RefHasher; + type Codec = ReferenceNodeCodec; } impl TrieConfiguration for ExtensionLayout { } @@ -76,7 +76,7 @@ impl TrieLayout for GenericNoExtensionLayout { impl TrieConfiguration for GenericNoExtensionLayout { } /// Trie layout without extension nodes. -pub type NoExtensionLayout = GenericNoExtensionLayout; +pub type NoExtensionLayout = GenericNoExtensionLayout; pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; pub type RefTrieDBNoExt<'a> = trie_db::TrieDB<'a, NoExtensionLayout>; @@ -93,19 +93,19 @@ pub type RefLookupNoExt<'a, Q> = trie_db::Lookup<'a, NoExtensionLayout, Q>; /// Typed version of hybrid_hash_node_adapter. pub fn hybrid_hash_node_adapter_no_ext( encoded_node: &[u8] -) -> std::result::Result::Out>, ()> { +) -> std::result::Result::Out>, ()> { trie_db::hybrid_hash_node_adapter::< - ReferenceNodeCodecNoExt, - KeccakHasher, + ReferenceNodeCodecNoExt, + RefHasher, > (encoded_node) } -pub fn reference_trie_root(input: I) -> ::Out where +pub fn reference_trie_root(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - trie_root::trie_root::(input) + trie_root::trie_root::(input) } /*fn reference_trie_root_unhashed(input: I) -> Vec where @@ -113,15 +113,15 @@ pub fn reference_trie_root(input: I) -> ::Out w A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - trie_root::unhashed_trie::(input) + trie_root::unhashed_trie::(input) }*/ -pub fn reference_trie_root_no_extension(input: I) -> ::Out where +pub fn reference_trie_root_no_extension(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - trie_root::trie_root_no_extension::(input) + trie_root::trie_root_no_extension::(input) } /*fn reference_trie_root_unhashed_no_extension(input: I) -> Vec where @@ -129,7 +129,7 @@ pub fn reference_trie_root_no_extension(input: I) -> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - trie_root::unhashed_trie_no_extension::(input) + trie_root::unhashed_trie_no_extension::(input) }*/ fn data_sorted_unique(input: I) -> Vec<(A, B)> @@ -143,22 +143,22 @@ fn data_sorted_unique(input: I) -> Vec<(A, B)> m.into_iter().collect() } -pub fn reference_trie_root_iter_build(input: I) -> ::Out where +pub fn reference_trie_root_iter_build(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootHybrid::::default(); + let mut cb = trie_db::TrieRootHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } -pub fn reference_trie_root_no_extension_iter_build(input: I) -> ::Out where +pub fn reference_trie_root_no_extension_iter_build(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootHybrid::::default(); + let mut cb = trie_db::TrieRootHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -168,7 +168,7 @@ fn reference_trie_root_unhashed_iter_build(input: I) -> Vec where A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -178,7 +178,7 @@ fn reference_trie_root_unhashed_no_extension_iter_build(input: I) -> Ve A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -743,9 +743,9 @@ impl ReferenceNodeCodec { } // NOTE: what we'd really like here is: -// `impl NodeCodec for RlpNodeCodec where ::Out: Decodable` +// `impl NodeCodec for RlpNodeCodec where ::Out: Decodable` // but due to the current limitations of Rust const evaluation we can't do -// `const HASHED_NULL_NODE: ::Out = ::Out( … … )`. +// `const HASHED_NULL_NODE: ::Out = ::Out( … … )`. // Perhaps one day soon? impl NodeCodec for ReferenceNodeCodec { type Error = CodecError; @@ -1333,7 +1333,7 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { } /// Compare trie builder and in memory trie. -pub fn compare_implementations + HashDBHybrid + Eq> ( +pub fn compare_implementations + HashDBHybrid + Eq> ( data: Vec<(Vec, Vec)>, mut memdb: X, mut hashdb: X, @@ -1379,10 +1379,10 @@ pub fn compare_implementations + Hash /// Compare trie builder and trie root implementations. pub fn compare_root( data: Vec<(Vec, Vec)>, - mut memdb: impl HashDBHybrid, + mut memdb: impl HashDBHybrid, ) { let root_new = { - let mut cb = TrieRootHybrid::::default(); + let mut cb = TrieRootHybrid::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1403,7 +1403,7 @@ pub fn compare_unhashed( data: Vec<(Vec, Vec)>, ) { let root_new = { - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1418,7 +1418,7 @@ pub fn compare_unhashed_no_extension( data: Vec<(Vec, Vec)>, ) { let root_new = { - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); + let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); trie_visit::(data.clone().into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; @@ -1430,13 +1430,13 @@ pub fn compare_unhashed_no_extension( /// Trie builder root calculation utility. pub fn calc_root( data: I, -) -> ::Out +) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRootHybrid::::default(); + let mut cb = TrieRootHybrid::::default(); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1445,13 +1445,13 @@ pub fn calc_root( /// This uses the variant without extension nodes. pub fn calc_root_no_extension( data: I, -) -> ::Out +) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRootHybrid::::default(); + let mut cb = TrieRootHybrid::::default(); trie_db::trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) } @@ -1460,12 +1460,12 @@ pub fn calc_root_no_extension( pub fn calc_root_build( data: I, hashdb: &mut DB -) -> ::Out +) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + HashDBHybrid, + DB: hash_db::HashDB + HashDBHybrid, { let mut cb = TrieBuilderHybrid::new(hashdb); trie_visit::(data.into_iter(), &mut cb); @@ -1477,12 +1477,12 @@ pub fn calc_root_build( pub fn calc_root_build_no_extension( data: I, hashdb: &mut DB, -) -> ::Out +) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + HashDBHybrid, + DB: hash_db::HashDB + HashDBHybrid, { let mut cb = TrieBuilderHybrid::new(hashdb); trie_db::trie_visit::(data.into_iter(), &mut cb); @@ -1493,8 +1493,8 @@ pub fn calc_root_build_no_extension( /// This uses the variant without extension nodes. pub fn compare_implementations_no_extension( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + HashDBHybrid, - mut hashdb: impl hash_db::HashDB + HashDBHybrid, + mut memdb: impl hash_db::HashDB + HashDBHybrid, + mut hashdb: impl hash_db::HashDB + HashDBHybrid, ) { let root_new = { let mut cb = TrieBuilderHybrid::new(&mut hashdb); @@ -1536,8 +1536,8 @@ pub fn compare_implementations_no_extension( /// ordering before running when trie_build expect correct ordering). pub fn compare_implementations_no_extension_unordered( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + HashDBHybrid, - mut hashdb: impl hash_db::HashDB + HashDBHybrid, + mut memdb: impl hash_db::HashDB + HashDBHybrid, + mut hashdb: impl hash_db::HashDB + HashDBHybrid, ) { let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { @@ -1581,7 +1581,7 @@ pub fn compare_implementations_no_extension_unordered( /// its input test data. pub fn compare_no_extension_insert_remove( data: Vec<(bool, Vec, Vec)>, - mut memdb: impl HashDBHybrid, + mut memdb: impl HashDBHybrid, ) { let mut data2 = std::collections::BTreeMap::new(); let mut root = Default::default(); @@ -1648,9 +1648,9 @@ mod tests { fn too_big_nibble_length() { // + 1 for 0 added byte of nibble encode let input = vec![0u8; (NIBBLE_SIZE_BOUND_NO_EXT as usize + 1) / 2 + 1]; - let enc = as NodeCodec> + let enc = as NodeCodec> ::leaf_node(((0, 0), &input), &[1]); - let dec = as NodeCodec> + let dec = as NodeCodec> ::decode(&enc).unwrap(); let o_sl = if let Node::Leaf(sl, _) = dec { Some(sl) diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 9261453d..0745da4d 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -26,7 +26,7 @@ use reference_trie::{ RefTrieDBMutNoExt, RefTrieDBNoExt, TrieDBIterator, - KeccakHasher, + RefHasher, }; use std::convert::TryInto; use trie_db::{DBValue, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut}; @@ -145,7 +145,7 @@ pub fn fuzz_that_compare_implementations(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data(input)); //println!("data:{:?}", &data); let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); + let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations(data, memdb, hashdb); } diff --git a/trie-db/src/fatdb.rs b/trie-db/src/fatdb.rs index f6e0cd57..7ec93c80 100644 --- a/trie-db/src/fatdb.rs +++ b/trie-db/src/fatdb.rs @@ -130,11 +130,11 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use crate::DBValue; - use reference_trie::{RefFatDBMut, RefFatDB, Trie, TrieMut, KeccakHasher}; + use reference_trie::{RefFatDBMut, RefFatDB, Trie, TrieMut, RefHasher}; #[test] fn fatdb_to_trie() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefFatDBMut::new(&mut memdb, &mut root); diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs index eb41da1c..3ada7235 100644 --- a/trie-db/src/fatdbmut.rs +++ b/trie-db/src/fatdbmut.rs @@ -112,11 +112,11 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use hash_db::{Hasher, EMPTY_PREFIX}; - use reference_trie::{RefFatDBMut, RefTrieDB, Trie, TrieMut, KeccakHasher}; + use reference_trie::{RefFatDBMut, RefTrieDB, Trie, TrieMut, RefHasher}; #[test] fn fatdbmut_to_trie() { - let mut memdb = MemoryDB::, _>::default(); + let mut memdb = MemoryDB::, _>::default(); let mut root = Default::default(); { let mut t = RefFatDBMut::new(&mut memdb, &mut root); @@ -124,19 +124,19 @@ mod test { } let t = RefTrieDB::new(&memdb, &root).unwrap(); assert_eq!( - t.get(&KeccakHasher::hash(&[0x01u8, 0x23])), + t.get(&RefHasher::hash(&[0x01u8, 0x23])), Ok(Some(vec![0x01u8, 0x23])), ); } #[test] fn fatdbmut_insert_remove_key_mapping() { - let mut memdb = MemoryDB::, _>::default(); + let mut memdb = MemoryDB::, _>::default(); let mut root = Default::default(); let key = [0x01u8, 0x23]; let val = [0x01u8, 0x24]; - let key_hash = KeccakHasher::hash(&key); - let aux_hash = KeccakHasher::hash(&key_hash); + let key_hash = RefHasher::hash(&key); + let aux_hash = RefHasher::hash(&key_hash); let mut t = RefFatDBMut::new(&mut memdb, &mut root); t.insert(&key, &val).unwrap(); assert_eq!(t.get(&key), Ok(Some(val.to_vec()))); diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index edc4b7b9..a55227ee 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -707,9 +707,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashe mod test { use crate::DBValue; use memory_db::{MemoryDB, HashKey, PrefixedKey}; - - type KeccakHasher = ordered_trie::OrderedTrieHasher; - + use reference_trie::RefHasher; #[test] fn trie_root_empty () { @@ -734,7 +732,7 @@ mod test { fn test_iter(data: Vec<(Vec, Vec)>) { use reference_trie::{RefTrieDBMut, TrieMut, RefTrieDB, Trie}; - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut db, &mut root); @@ -760,7 +758,7 @@ mod test { fn test_iter_no_extension(data: Vec<(Vec, Vec)>) { use reference_trie::{RefTrieDBMutNoExt, TrieMut, RefTrieDBNoExt, Trie}; - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMutNoExt::new(&mut db, &mut root); @@ -794,27 +792,27 @@ mod test { fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); + let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations(data, memdb, hashdb); } fn compare_implementations_h(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); + let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations(data, memdb, hashdb); } fn compare_implementations_no_extension(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); + let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations_no_extension(data, memdb, hashdb); } fn compare_implementations_no_extension_prefixed(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); + let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations_no_extension(data, memdb, hashdb); } fn compare_implementations_no_extension_unordered(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); + let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations_no_extension_unordered(data, memdb, hashdb); } fn compare_no_extension_insert_remove(data: Vec<(bool, Vec, Vec)>) { diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index a0d350fa..2e1b5ec1 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -391,14 +391,14 @@ mod tests { use reference_trie::{ RefTrieDB, RefTrieDBMut, TrieError, TrieMut, TrieIterator, TrieDBNodeIterator, NibbleSlice, NibbleVec, - node::Node, KeccakHasher, + node::Node, RefHasher, }; use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt}; - type MemoryDB = memory_db::MemoryDB, DBValue>; + type MemoryDB = memory_db::MemoryDB, DBValue>; fn build_trie_db_with_extension(pairs: &[(Vec, Vec)]) - -> (MemoryDB, ::Out) + -> (MemoryDB, ::Out) { let mut memdb = MemoryDB::default(); let mut root = Default::default(); @@ -412,7 +412,7 @@ mod tests { } fn build_trie_db_without_extension(pairs: &[(Vec, Vec)]) - -> (MemoryDB, ::Out) + -> (MemoryDB, ::Out) { let mut memdb = MemoryDB::default(); let mut root = Default::default(); diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 99356614..1a899f29 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -407,13 +407,13 @@ pub fn hybrid_hash_node_adapter, H #[test] fn test_hybrid_hash_node_adapter() { - use reference_trie::{RefTrieDBMutNoExt, TrieMut, hybrid_hash_node_adapter_no_ext, KeccakHasher}; + use reference_trie::{RefTrieDBMutNoExt, TrieMut, hybrid_hash_node_adapter_no_ext, RefHasher}; use crate::DBValue; use memory_db::{MemoryDB, HashKey}; use hash_db::EMPTY_PREFIX; - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let data = vec![ (vec![0], vec![251; 34]), (vec![0, 1], vec![252; 34]), @@ -429,7 +429,7 @@ fn test_hybrid_hash_node_adapter() { t.insert(key, value).unwrap(); } } - let mut db2 = MemoryDB::, DBValue>::default(); + let mut db2 = MemoryDB::, DBValue>::default(); for (_key, (value, rc)) in db.clone().drain() { if rc > 0 { if !db2.insert_hybrid( diff --git a/trie-db/src/recorder.rs b/trie-db/src/recorder.rs index ea4c18fd..69246623 100644 --- a/trie-db/src/recorder.rs +++ b/trie-db/src/recorder.rs @@ -79,7 +79,7 @@ impl Recorder { mod tests { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use reference_trie::{RefTrieDB, RefTrieDBMut, Trie, TrieMut, Recorder, Record, KeccakHasher}; + use reference_trie::{RefTrieDB, RefTrieDBMut, Trie, TrieMut, Recorder, Record, RefHasher}; #[test] fn basic_recorder() { @@ -88,7 +88,7 @@ mod tests { let node1 = vec![1, 2, 3, 4]; let node2 = vec![4, 5, 6, 7, 8, 9, 10]; - let (hash1, hash2) = (KeccakHasher::hash(&node1), KeccakHasher::hash(&node2)); + let (hash1, hash2) = (RefHasher::hash(&node1), RefHasher::hash(&node2)); basic.record(&hash1, &node1, 0); basic.record(&hash2, &node2, 456); @@ -115,8 +115,8 @@ mod tests { let node1 = vec![1, 2, 3, 4]; let node2 = vec![4, 5, 6, 7, 8, 9, 10]; - let hash1 = KeccakHasher::hash(&node1); - let hash2 = KeccakHasher::hash(&node2); + let hash1 = RefHasher::hash(&node1); + let hash2 = RefHasher::hash(&node2); basic.record(&hash1, &node1, 0); basic.record(&hash2, &node2, 456); @@ -133,7 +133,7 @@ mod tests { // #[test] TODO put it back for reftriedb not complex fn trie_record() { - let mut db = MemoryDB::, _>::default(); + let mut db = MemoryDB::, _>::default(); let mut root = Default::default(); { let mut x = RefTrieDBMut::new(&mut db, &mut root); diff --git a/trie-db/src/sectriedb.rs b/trie-db/src/sectriedb.rs index 82d7160f..99c05341 100644 --- a/trie-db/src/sectriedb.rs +++ b/trie-db/src/sectriedb.rs @@ -87,16 +87,16 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use reference_trie::{RefTrieDBMut, RefSecTrieDB, Trie, TrieMut, KeccakHasher}; + use reference_trie::{RefTrieDBMut, RefSecTrieDB, Trie, TrieMut, RefHasher}; use crate::DBValue; #[test] fn trie_to_sectrie() { - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut db, &mut root); - t.insert(&KeccakHasher::hash(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); + t.insert(&RefHasher::hash(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); } let t = RefSecTrieDB::new(&db, &root).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs index 00596822..14fc7840 100644 --- a/trie-db/src/sectriedbmut.rs +++ b/trie-db/src/sectriedbmut.rs @@ -93,12 +93,12 @@ where mod test { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use reference_trie::{RefTrieDB, RefSecTrieDBMut, Trie, TrieMut, KeccakHasher}; + use reference_trie::{RefTrieDB, RefSecTrieDBMut, Trie, TrieMut, RefHasher}; use crate::DBValue; #[test] fn sectrie_to_trie() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefSecTrieDBMut::new(&mut memdb, &mut root); @@ -106,7 +106,7 @@ mod test { } let t = RefTrieDB::new(&memdb, &root).unwrap(); assert_eq!( - t.get(&KeccakHasher::hash(&[0x01u8, 0x23])).unwrap().unwrap(), + t.get(&RefHasher::hash(&[0x01u8, 0x23])).unwrap().unwrap(), vec![0x01u8, 0x23], ); } diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index b682164d..cddaa538 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -35,11 +35,11 @@ use crate::rstd::{fmt, vec::Vec}; /// # Example /// ``` /// use hash_db::Hasher; -/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut, KeccakHasher}; +/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut, RefHasher}; /// use trie_db::DBValue; /// use memory_db::*; /// -/// let mut memdb = MemoryDB::, _>::default(); +/// let mut memdb = MemoryDB::, _>::default(); /// let mut root = Default::default(); /// RefTrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); /// let t = RefTrieDB::new(&memdb, &root).unwrap(); @@ -333,7 +333,7 @@ mod tests { use memory_db::{MemoryDB, PrefixedKey}; use crate::DBValue; use reference_trie::{RefTrieDB, RefTrieDBMut, RefLookup, Trie, TrieMut, NibbleSlice}; - use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt, KeccakHasher}; + use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt, RefHasher}; use hex_literal::hex; #[test] @@ -343,7 +343,7 @@ mod tests { (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut memdb, &mut root); @@ -399,7 +399,7 @@ mod tests { (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut memdb, &mut root); @@ -480,7 +480,7 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut memdb, &mut root); @@ -511,7 +511,7 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); @@ -537,7 +537,7 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); @@ -576,7 +576,7 @@ mod tests { #[test] fn get_length_with_extension() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut memdb, &mut root); @@ -592,7 +592,7 @@ mod tests { #[test] fn get_length_without_extension() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); @@ -615,7 +615,7 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let root = { let mut t = RefTrieDBMut::new(&mut memdb, &mut root); @@ -683,7 +683,7 @@ mod tests { #[test] fn test_lookup_with_corrupt_data_returns_decoder_error() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut memdb, &mut root); diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index a90345b2..fcf691a9 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -422,15 +422,15 @@ impl<'a, H> Index<&'a StorageHandle> for NodeStorage { /// # Example /// ``` /// use hash_db::Hasher; -/// use reference_trie::{RefTrieDBMut, TrieMut, KeccakHasher}; +/// use reference_trie::{RefTrieDBMut, TrieMut, RefHasher}; /// use trie_db::DBValue; /// use memory_db::*; /// -/// let mut memdb = MemoryDB::, DBValue>::default(); +/// let mut memdb = MemoryDB::, DBValue>::default(); /// let mut root = Default::default(); /// let mut t = RefTrieDBMut::new(&mut memdb, &mut root); /// assert!(t.is_empty()); -/// assert_eq!(*t.root(), KeccakHasher::hash(&[0u8][..])); +/// assert_eq!(*t.root(), RefHasher::hash(&[0u8][..])); /// t.insert(b"foo", b"bar").unwrap(); /// assert!(t.contains(b"foo").unwrap()); /// assert_eq!(t.get(b"foo").unwrap().unwrap(), b"bar".to_vec()); @@ -1678,13 +1678,13 @@ mod tests { use memory_db::{MemoryDB, PrefixedKey}; use hash_db::Hasher; use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, - ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, KeccakHasher, + ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, RefHasher, reference_trie_root_no_extension_iter_build as reference_trie_root_no_extension}; use crate::nibble::BackingByteVec; fn populate_trie<'db>( - db: &'db mut dyn HashDBHybridDyn, - root: &'db mut ::Out, + db: &'db mut dyn HashDBHybridDyn, + root: &'db mut ::Out, v: &[(Vec, Vec)] ) -> RefTrieDBMut<'db> { let mut t = RefTrieDBMut::new(db, root); @@ -1704,8 +1704,8 @@ mod tests { } fn populate_trie_no_extension<'db>( - db: &'db mut dyn HashDBHybridDyn, - root: &'db mut ::Out, + db: &'db mut dyn HashDBHybridDyn, + root: &'db mut ::Out, v: &[(Vec, Vec)] ) -> RefTrieDBMutNoExt<'db> { let mut t = RefTrieDBMutNoExt::new(db, root); @@ -1724,8 +1724,8 @@ mod tests { } } - fn reference_hashed_null_node() -> ::Out { - as NodeCodec>::hashed_null_node() + fn reference_hashed_null_node() -> ::Out { + as NodeCodec>::hashed_null_node() } #[test] @@ -1745,7 +1745,7 @@ mod tests { }.make_with(&mut seed); let real = reference_trie_root(x.clone()); - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut memtrie = populate_trie(&mut memdb, &mut root, &x); @@ -1788,7 +1788,7 @@ mod tests { }.make_with(&mut seed); let real = reference_trie_root_no_extension(x.clone()); - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut memtrie = populate_trie_no_extension(&mut memdb, &mut root, &x); @@ -1819,7 +1819,7 @@ mod tests { #[test] fn init() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); let hashed_null_node = reference_hashed_null_node(); @@ -1828,7 +1828,7 @@ mod tests { #[test] fn insert_on_empty() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1842,7 +1842,7 @@ mod tests { fn remove_to_empty() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); @@ -1877,7 +1877,7 @@ mod tests { #[test] fn insert_replace_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1890,7 +1890,7 @@ mod tests { #[test] fn insert_make_branch_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1903,7 +1903,7 @@ mod tests { #[test] fn insert_into_branch_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1918,7 +1918,7 @@ mod tests { #[test] fn insert_value_into_branch_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1931,7 +1931,7 @@ mod tests { #[test] fn insert_split_leaf() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1944,7 +1944,7 @@ mod tests { #[test] fn insert_split_extenstion() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); @@ -1962,7 +1962,7 @@ mod tests { let big_value0 = b"00000000000000000000000000000000"; let big_value1 = b"11111111111111111111111111111111"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); @@ -1977,7 +1977,7 @@ mod tests { fn insert_duplicate_value() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value).unwrap(); @@ -1990,7 +1990,7 @@ mod tests { #[test] fn test_at_empty() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let t = RefTrieDBMut::new(&mut memdb, &mut root); assert_eq!(t.get(&[0x5]).unwrap(), None); @@ -1998,7 +1998,7 @@ mod tests { #[test] fn test_at_one() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -2009,7 +2009,7 @@ mod tests { #[test] fn test_at_three() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -2039,12 +2039,12 @@ mod tests { }.make_with(&mut seed); let real = reference_trie_root(x.clone()); - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut memtrie = populate_trie(&mut memdb, &mut root, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let mut memdb2 = MemoryDB::, DBValue>::default(); + let mut memdb2 = MemoryDB::, DBValue>::default(); let mut root2 = Default::default(); let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { @@ -2066,7 +2066,7 @@ mod tests { #[test] fn test_trie_existing() { - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { let mut t = RefTrieDBMut::new(&mut db, &mut root); @@ -2089,7 +2089,7 @@ mod tests { count: 4, }.make_with(&mut seed); - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { @@ -2118,7 +2118,7 @@ mod tests { count: 2, }.make_with(&mut seed); - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let mut t = RefTrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { From 6b89dc78a8305e44cc9660a34bda227c406f2b08 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 12 May 2020 15:42:19 +0200 Subject: [PATCH 33/73] Using internal hasher instead of buffer. --- hash-db/src/lib.rs | 57 ++++++++++++---------- memory-db/src/lib.rs | 2 + ordered-trie/src/lib.rs | 69 ++++++++++++++------------- test-support/keccak-hasher/src/lib.rs | 21 +++++++- trie-db/src/iter_build.rs | 28 ++++++----- trie-db/src/node_codec.rs | 10 +++- trie-db/src/proof/generate.rs | 2 +- trie-db/src/proof/verify.rs | 4 ++ trie-db/src/trie_codec.rs | 8 +++- trie-db/src/triedbmut.rs | 15 +++++- 10 files changed, 140 insertions(+), 76 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 3041bb0f..256c5532 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -68,34 +68,41 @@ pub trait Hasher: Sync + Send { pub trait BinaryHasher: Hasher { /// Hash for the empty content (is hash(&[])). const NULL_HASH: &'static [u8]; - type Buffer: AsRef<[u8]> + AsMut<[u8]> + Default; + + /// State buffer for hashing. + type Buffer; + + fn init_buffer() -> Self::Buffer; + fn reset_buffer(buf: &mut Self::Buffer); + fn buffer_hash(buff: &mut Self::Buffer, x: &[u8]); + + /// After calling `buffer_finalize`, one do not have to call `reset_buffer`. + fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out; } -/// Test function to use on every binary buffer implementation. +/// Test function to use on any binary buffer implementation. pub fn test_binary_hasher() { let size = ::LENGTH * 2; - let buf = ::Buffer::default(); - assert_eq!(buf.as_ref().len(), size); + let half_size = ::LENGTH / 2; + let mut val = vec![0u8; size]; + val[0] = 1; + let mut buf = ::init_buffer(); + H::buffer_hash(&mut buf, &val[..half_size]); + H::buffer_hash(&mut buf, &val[half_size..::LENGTH]); + let three = core::cmp::min(3, half_size); + H::buffer_hash(&mut buf, &val[::LENGTH..::LENGTH + three]); + H::buffer_hash(&mut buf, &val[::LENGTH + three..]); + let h = H::buffer_finalize(&mut buf); + let h2 = H::hash(&val[..]); + assert_eq!(h, h2); + H::buffer_hash(&mut buf, &val[..]); + let h = H::buffer_finalize(&mut buf); + assert_eq!(h, h2); let null_hash = H::hash(&[]); + H::reset_buffer(&mut buf); + let null_hash2 = H::buffer_finalize(&mut buf); assert_eq!(H::NULL_HASH, null_hash.as_ref()); -} - -/// A buffer for binary hasher of size 64. -pub struct Buffer64([u8; 64]); -impl AsRef<[u8]> for Buffer64 { - fn as_ref(&self) -> &[u8] { - &self.0[..] - } -} -impl AsMut<[u8]> for Buffer64 { - fn as_mut(&mut self) -> &mut [u8] { - &mut self.0[..] - } -} -impl Default for Buffer64 { - fn default() -> Self { - Buffer64([0; 64]) - } + assert_eq!(H::NULL_HASH, null_hash2.as_ref()); } /// Trait modelling a plain datastore whose key is a fixed type. @@ -222,7 +229,7 @@ impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { /// Same as HashDB but can modify the value upon storage, and apply /// `HasherHybrid`. -pub trait HashDBHybrid: Send + Sync + HashDB { +pub trait HashDBHybrid: Send + Sync + HashDB { /// `HashDB` is often use to load content from encoded node. /// This will not preserve insertion done through `insert_branch_hybrid` calls /// and break the proof. @@ -253,14 +260,13 @@ pub trait HashDBHybrid: Send + Sync + HashDB { children: I, additional_hashes: I2, proof: bool, + buffer: &mut H::Buffer, ) -> H::Out; } pub trait HasherHybrid: BinaryHasher { /// Alternate hash with hybrid proof allowed - /// TODO expose buffer !! (then memory db use a single buf) - /// TODO EMCH also split depending on proof or not!! fn hash_hybrid< I: Iterator::Out>>, I2: Iterator::Out>, @@ -270,5 +276,6 @@ pub trait HasherHybrid: BinaryHasher { children: I, additional_hashes: I2, proof: bool, + buffer: &mut Self::Buffer, ) -> Option; } diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index ad61b52b..813e2efb 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -639,6 +639,7 @@ where children: I, additional_hashes: I2, proof: bool, + buff: &mut H::Buffer, ) -> H::Out { if T::from(value) == self.null_node_data { return self.hashed_null_node.clone(); @@ -650,6 +651,7 @@ where children, additional_hashes, proof, + buff, ) { key } else { diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 8b09a121..727f5426 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -731,14 +731,11 @@ pub trait ProcessNodeProof: ProcessNode { /// Does only proccess hash on its buffer. /// Correct buffer size is expected, this is unchecked. -pub struct HashOnly<'a, H>(&'a mut [u8], PhantomData); +pub struct HashOnly<'a, H: BinaryHasher>(&'a mut H::Buffer); impl<'a, H: BinaryHasher> HashOnly<'a, H> { pub fn new(buff: &'a mut H::Buffer) -> Self { - HashOnly(buff.as_mut(), PhantomData) - } - pub fn new_unchecked(buff: &'a mut [u8]) -> Self { - HashOnly(buff, PhantomData) + HashOnly(buff) } } @@ -747,18 +744,16 @@ impl<'a, H: BinaryHasher, KN> ProcessNode for HashOnly<'a, H> { H::NULL_HASH } fn process(&mut self, _key: &KN, child1: &[u8], child2: &[u8]) -> H::Out { - // Should use lower level trait than Hasher to avoid copies. - self.0[..H::LENGTH].copy_from_slice(child1); - self.0[H::LENGTH..].copy_from_slice(child2); - H::hash(&self.0[..]) + H::reset_buffer(&mut self.0); + H::buffer_hash(&mut self.0, child1); + H::buffer_hash(&mut self.0, child2); + H::buffer_finalize(&mut self.0) } fn register_root(&mut self, _root: &H::Out) { } } -/// Buffer length need to be right and is unchecked, proof elements are -/// stored in memory (no streaming). -pub struct HashProof<'a, H: Hasher, I, KN> { - buffer: &'a mut [u8], +pub struct HashProof<'a, H: BinaryHasher, I, KN> { + buffer: &'a mut H::Buffer, // `I` is a iterator over the depth of every element included in the proof. // Can be obtain from `iter_path_node_key` filtered by contained elements. to_prove: I, @@ -916,13 +911,13 @@ impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> HashProof<'a, H, // with either depth_at or the depth_iterator skipping undesired elements (second // seems better as it filters out of range. pub fn new( - buff: &'a mut H::Buffer, + buffer: &'a mut H::Buffer, mut to_prove: I, additional_hash_only: bool, ) -> Self { let state = MultiProofState::new(&mut to_prove); HashProof { - buffer: buff.as_mut(), + buffer, to_prove, state, additional_hash: Vec::new(), @@ -962,9 +957,8 @@ impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> ProcessNode::new(self.buffer); + h.process(key, child1, child2) } } fn register_root(&mut self, root: &H::Out) { @@ -1142,11 +1136,11 @@ impl HasherHybrid for OrderedTrieHasher { children: I, additional_hashes: I2, proof: bool, + buffer: &mut H::Buffer, ) -> Option { let seq_trie = SequenceBinaryTree::new(0, 0, nb_children); - let mut hash_buf2 = ::Buffer::default(); - let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); + let mut callback_read_proof = HashOnly::::new(buffer); let hash = if !proof { // full node let iter = children.filter_map(|v| v); // TODO assert same number as count @@ -1174,17 +1168,28 @@ impl HasherHybrid for OrderedTrieHasher { return None; } }; - // TODO really need a real hash trait - let mut buf = Vec::with_capacity(header.len() + hash.as_ref().len()); - buf.extend_from_slice(header); - buf.extend_from_slice(hash.as_ref()); - Some(H::hash(buf.as_slice())) + let mut hash_buf2 = ::init_buffer(); + ::buffer_hash(&mut hash_buf2, header); + ::buffer_hash(&mut hash_buf2, hash.as_ref()); + Some(H::buffer_finalize(&mut hash_buf2)) } } impl BinaryHasher for OrderedTrieHasher { const NULL_HASH: &'static [u8] = ::NULL_HASH; type Buffer = ::Buffer; + fn init_buffer() -> Self::Buffer { + ::init_buffer() + } + fn reset_buffer(buf: &mut Self::Buffer) { + ::reset_buffer(buf) + } + fn buffer_hash(buff: &mut Self::Buffer, x: &[u8]) { + ::buffer_hash(buff, x) + } + fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out { + ::buffer_finalize(buff) + } } impl Hasher for OrderedTrieHasher { @@ -1416,7 +1421,7 @@ mod test { let result = base16_roots(); for l in 0..17 { let tree = Tree::new(0, 0, l); - let mut hash_buf = ::Buffer::default(); + let mut hash_buf = ::init_buffer(); let mut callback = HashOnly::::new(&mut hash_buf); let hashes: Vec<_> = hashes(l); let root = trie_root::<_, UsizeKeyNode, _, _>(&tree, hashes.into_iter(), &mut callback); @@ -1429,9 +1434,9 @@ mod test { let result = base16_roots(); for l in 0..17 { let tree = Tree::new(0, 0, l); - let mut hash_buf = ::Buffer::default(); - let mut hash_buf2 = ::Buffer::default(); - let mut hash_buf3 = ::Buffer::default(); + let mut hash_buf = ::init_buffer(); + let mut hash_buf2 = ::init_buffer(); + let mut hash_buf3 = ::init_buffer(); let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); let hashes: Vec<_> = hashes(l); for p in 0..l { @@ -1483,9 +1488,9 @@ mod test { let l = *l; let ps = *ps; let tree = Tree::new(0, 0, l); - let mut hash_buf = ::Buffer::default(); - let mut hash_buf2 = ::Buffer::default(); - let mut hash_buf3 = ::Buffer::default(); + let mut hash_buf = ::init_buffer(); + let mut hash_buf2 = ::init_buffer(); + let mut hash_buf3 = ::init_buffer(); let mut callback_read_proof = HashOnly::::new(&mut hash_buf2); let hashes: Vec<_> = hashes(l); let mut to_prove = Vec::new(); diff --git a/test-support/keccak-hasher/src/lib.rs b/test-support/keccak-hasher/src/lib.rs index ea3e3234..cb7a4aec 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/test-support/keccak-hasher/src/lib.rs @@ -40,7 +40,26 @@ impl BinaryHasher for KeccakHasher { const NULL_HASH: &'static [u8] = &[197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112]; - type Buffer = hash_db::Buffer64; + type Buffer = Keccak; + + fn init_buffer() -> Self::Buffer { + Keccak::new_keccak256() + } + + fn reset_buffer(buff: &mut Self::Buffer) { + let _ = core::mem::replace(buff, Self::init_buffer()); + } + + fn buffer_hash(buff: &mut Self::Buffer, x: &[u8]) { + buff.update(&x[..]) + } + + fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out { + let mut res: [u8; 32] = [0; 32]; + let k = core::mem::replace(buff, Self::init_buffer()); + k.finalize(&mut res); + res + } } #[test] diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index a55227ee..03d7e4ba 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -17,7 +17,7 @@ //! implementation. //! See `trie_visit` function. -use hash_db::{Hasher, HashDB, Prefix, HasherHybrid, HashDBHybrid}; +use hash_db::{Hasher, HashDB, Prefix, HasherHybrid, HashDBHybrid, BinaryHasher}; use crate::rstd::{cmp::max, marker::PhantomData, vec::Vec, EmptyIter, ops::Range}; use crate::triedbmut::{ChildReference}; use crate::nibble::NibbleSlice; @@ -395,15 +395,16 @@ impl<'a, H, HO, V, DB> TrieBuilder<'a, H, HO, V, DB> { /// Get trie root and insert visited node in a hash_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). -pub struct TrieBuilderHybrid<'a, H, HO, V, DB> { +pub struct TrieBuilderHybrid<'a, H: BinaryHasher, HO, V, DB> { db: &'a mut DB, pub root: Option, + buffer: H::Buffer, _ph: PhantomData<(H, V)>, } -impl<'a, H, HO, V, DB> TrieBuilderHybrid<'a, H, HO, V, DB> { +impl<'a, H: BinaryHasher, HO, V, DB> TrieBuilderHybrid<'a, H, HO, V, DB> { pub fn new(db: &'a mut DB) -> Self { - TrieBuilderHybrid { db, root: None, _ph: PhantomData } + TrieBuilderHybrid { db, root: None, buffer: H::init_buffer(), _ph: PhantomData } } } @@ -465,6 +466,7 @@ impl<'a, H: HasherHybrid, V, DB: HashDBHybrid> ProcessEncodedNode< ProcessEncodedNode<::Out> for TrieRoot { +pub struct TrieRootHybrid { /// The resulting root. pub root: Option, - _ph: PhantomData, + buffer: H::Buffer, } -impl Default for TrieRootHybrid { +impl Default for TrieRootHybrid { fn default() -> Self { - TrieRootHybrid { root: None, _ph: PhantomData } + TrieRootHybrid { root: None, buffer: H::init_buffer() } } } @@ -554,6 +556,7 @@ impl ProcessEncodedNode<::Out> for TrieRootHybrid< iter, EmptyIter::default(), false, + &mut self.buffer, ).expect("only proof fails: TODO two primitives") } else { ::hash(&encoded_node[..]) @@ -581,15 +584,15 @@ impl Default for TrieRootUnhashed { } /// Get the trie root node encoding. -pub struct TrieRootUnhashedHybrid { +pub struct TrieRootUnhashedHybrid { /// The resulting encoded root. pub root: Option>, - _ph: PhantomData, + buffer: H::Buffer, } -impl Default for TrieRootUnhashedHybrid { +impl Default for TrieRootUnhashedHybrid { fn default() -> Self { - TrieRootUnhashedHybrid { root: None, _ph: PhantomData } + TrieRootUnhashedHybrid { root: None, buffer: H::init_buffer() } } } @@ -690,6 +693,7 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashe iter, EmptyIter::default(), false, + &mut self.buffer, ).expect("only proof fails: TODO two primitives") } else { ::hash(&encoded_node[..]) diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 1a899f29..780efcb7 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -18,7 +18,7 @@ use crate::MaybeDebug; use crate::node::{Node, NodePlan}; use crate::ChildReference; -use hash_db::HasherHybrid; +use hash_db::{HasherHybrid, BinaryHasher}; use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range}; @@ -205,7 +205,7 @@ impl ChildProofHeader { use hash_db::{HashDB, Prefix, HashDBRef, Hasher, HashDBHybrid}; -pub trait HashDBHybridDyn: Send + Sync + HashDB { +pub trait HashDBHybridDyn: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. @@ -218,6 +218,7 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { value: &[u8], children: &[Option>], common: ChildProofHeader, + buffer: &mut H::Buffer, ) -> H::Out; } @@ -228,6 +229,7 @@ impl> HashDBHybridDyn for C { value: &[u8], children: &[Option>], common: ChildProofHeader, + buffer: &mut H::Buffer, ) -> H::Out { // TODO factor this with iter_build (just use the trait) also use in adapter from codec @@ -247,6 +249,7 @@ impl> HashDBHybridDyn for C { children, EmptyIter::default(), false, + buffer, ) } } @@ -377,6 +380,7 @@ impl Iterator for HashesPlan { } /// Adapter standard implementation to use with `HashDBInsertComplex` function. +/// TODO EMCH seems unused (there is test but??) pub fn hybrid_hash_node_adapter, Hasher: HasherHybrid>( encoded_node: &[u8] ) -> crate::rstd::result::Result, ()> { @@ -390,12 +394,14 @@ pub fn hybrid_hash_node_adapter, H dest.as_mut()[..range.len()].copy_from_slice(&encoded_node[range]); dest })); + let mut buf = Hasher::init_buffer(); Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( common.header(encoded_node), nb_children, children, EmptyIter::default(), false, // not a proof, will not fail + &mut buf, ).expect("not a proof, does not fail")) // TODO EMCH split function in two variants! }, _ => unreachable!("hybrid only touch branch node"), diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 8525b331..0a2dece8 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -268,7 +268,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) .collect::>(); keys.sort(); keys.dedup(); - let mut hash_buf = ::Buffer::default(); + let mut hash_buf = ::init_buffer(); let hash_buf = &mut hash_buf; diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index 15c63425..b808286b 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -457,6 +457,9 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve false, L::HYBRID_HASH, )?; + let mut hybrid_buf = if L::HYBRID_HASH { + Some(L::Hash::init_buffer()) + } else { None }; loop { // Insert omitted value. match last_entry.advance_item(&mut items_iter)? { @@ -509,6 +512,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve children, additional_hash.into_iter(), true, + hybrid_buf.as_mut().expect("Initialized for hybrid above"), ) { h } else { diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 85d9b2ae..1de82386 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -25,7 +25,7 @@ //! expected to save roughly (n - 1) hashes in size where n is the number of nodes in the partial //! trie. -use hash_db::{HasherHybrid, HashDBHybrid}; +use hash_db::{HasherHybrid, HashDBHybrid, BinaryHasher}; use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieHash, TrieError, TrieDB, TrieDBNodeIterator, TrieLayout, NodeCodecHybrid, @@ -209,7 +209,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE let mut output = Vec::new(); // TODO make it optional and replace boolean is_hybrid - let mut hash_buf = ::Buffer::default(); + let mut hash_buf = ::init_buffer(); let hash_buf = &mut hash_buf; // The stack of nodes through a path in the trie. Each entry is a child node of the preceding @@ -508,6 +508,9 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) _marker: PhantomData::default(), }; + let mut hybrid_buf = if L::HYBRID_HASH { + Some(L::Hash::init_buffer()) + } else { None }; loop { if !last_entry.advance_child_index()? { last_entry.push_to_prefix(&mut prefix); @@ -552,6 +555,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) children, additional_hashes, true, + hybrid_buf.as_mut().expect("Initialized for hybrid above"), ) } else { db.insert(prefix.as_prefix(), node_data.as_ref()) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index fcf691a9..d1654d8e 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -20,7 +20,7 @@ use super::lookup::Lookup; use super::node::{NodeHandle as EncodedNodeHandle, Node as EncodedNode, decode_hash}; use crate::node_codec::HashDBHybridDyn; -use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; +use hash_db::{Hasher, Prefix, EMPTY_PREFIX, BinaryHasher}; use hashbrown::HashSet; use crate::node_codec::{NodeCodec, NodeCodecHybrid, ChildProofHeader}; @@ -449,6 +449,7 @@ where /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, + hybrid_hash_buffer: Option<::Buffer>, } impl<'a, L> TrieDBMut<'a, L> @@ -479,6 +480,13 @@ where root_handle, death_row: HashSet::new(), hash_count: 0, + hybrid_hash_buffer: None, + } + } + + fn hybrid_hash_buffer_lazy_init(&mut self) { + if self.hybrid_hash_buffer.is_none() { + self.hybrid_hash_buffer = Some(L::Hash::init_buffer()) } } @@ -500,6 +508,7 @@ where root_handle, death_row: HashSet::new(), hash_count: 0, + hybrid_hash_buffer: None, }) } /// Get the backing database. @@ -1472,11 +1481,13 @@ where #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:#x?}", &encoded_root[..]); if let Some(children) = register_children { + self.hybrid_hash_buffer_lazy_init(); *self.root = self.db.insert_branch_hybrid( EMPTY_PREFIX, &encoded_root[..], &children[..], no_node, + self.hybrid_hash_buffer.as_mut().expect("Lazy init above"), ); } else { *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root[..]); @@ -1530,11 +1541,13 @@ where }; if encoded.len() >= L::Hash::LENGTH { let hash = if let Some(children) = register_children { + self.hybrid_hash_buffer_lazy_init(); self.db.insert_branch_hybrid( prefix.as_prefix(), &encoded[..], &children[..], no_node, + self.hybrid_hash_buffer.as_mut().expect("Lazy init above"), ) } else { self.db.insert(prefix.as_prefix(), &encoded[..]) From a6c2bc362f5f8478260b6a24522248b81fcff1c2 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 12 May 2020 16:30:28 +0200 Subject: [PATCH 34/73] Running test on blake2b like in substrate. --- test-support/reference-trie/Cargo.toml | 2 + test-support/reference-trie/src/lib.rs | 56 +++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 2eec20b1..80757d0a 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -15,6 +15,7 @@ trie-db = { path = "../../trie-db", default-features = false, version = "0.20.0" trie-root = { path = "../../trie-root", default-features = false, version = "0.16.0" } ordered-trie = { path = "../../ordered-trie", default-features = false, version = "0.19.2"} parity-scale-codec = { version = "1.0.3", features = ["derive"] } +blake2-rfc = { version = "0.2.18", default-features = false } [dev-dependencies] trie-bench = { path = "../trie-bench", version = "0.21.0" } @@ -30,4 +31,5 @@ default = ["std"] std = [ "trie-db/std", "trie-root/std", + "blake2-rfc/std", ] diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index a89cf741..89ab4083 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -48,7 +48,8 @@ pub mod node { } /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. -pub type RefHasher = ordered_trie::OrderedTrieHasher; +pub type RefHasher = ordered_trie::OrderedTrieHasher; +//pub type RefHasher = ordered_trie::OrderedTrieHasher; /// Trie layout using extension nodes. pub struct ExtensionLayout; @@ -1683,3 +1684,56 @@ mod tests { } } } + +pub mod blake2 { + use hash_db::{Hasher, BinaryHasher}; + use hash256_std_hasher::Hash256StdHasher; + + /// Concrete implementation of Hasher using Blake2b 256-bit hashes + #[derive(Debug)] + pub struct Blake2Hasher; + + impl Hasher for Blake2Hasher { + type Out = [u8; 32]; + type StdHasher = Hash256StdHasher; + const LENGTH: usize = 32; + + fn hash(x: &[u8]) -> Self::Out { + let mut dest = [0u8; 32]; + dest.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], x).as_bytes()); + dest + } + } + + impl BinaryHasher for Blake2Hasher { + const NULL_HASH: &'static [u8] = &[14, 87, 81, 192, 38, 229, + 67, 178, 232, 171, 46, 176, 96, 153, 218, 161, 209, 229, 223, + 71, 119, 143, 119, 135, 250, 171, 69, 205, 241, 47, 227, 168]; + type Buffer = blake2_rfc::blake2b::Blake2b; + + fn init_buffer() -> Self::Buffer { + blake2_rfc::blake2b::Blake2b::new(32) + } + + fn reset_buffer(buff: &mut Self::Buffer) { + let _ = core::mem::replace(buff, Self::init_buffer()); + } + + fn buffer_hash(buff: &mut Self::Buffer, x: &[u8]) { + buff.update(&x[..]) + } + + fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out { + let mut res: [u8; 32] = [0; 32]; + let k = core::mem::replace(buff, Self::init_buffer()); + res.copy_from_slice(k.finalize().as_bytes()); + res + } + + } + + #[test] + fn test_blake2b_hasher() { + hash_db::test_binary_hasher::() + } +} From 3439dad2f58fd827fcf055302346ea658c96218a Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 12 May 2020 17:05:30 +0200 Subject: [PATCH 35/73] Clean up unused code in unordered trie --- ordered-trie/src/lib.rs | 468 +++++++++++++------------------------- trie-db/src/trie_codec.rs | 2 +- 2 files changed, 158 insertions(+), 312 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 727f5426..9e1ebac9 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -47,71 +47,6 @@ use crate::rstd::marker::PhantomData; pub type DBValue = Vec; -pub mod key { - /// Base type for key, TODO - /// implementing on usize for now, - /// then this will probably look like - /// substrate simple arithmetic trait. - /// -> need an implementation for [u8] - /// (unbounded key length) - pub trait OrderedKey: Ord {} - - impl OrderedKey for T {} -} - - - -pub mod meta { - use hash_db::Hasher; - use crate::rstd::vec::Vec; - - /// Codec for the meta needed to get - /// information for the tree. - pub trait MetaCodec { - /// If false we do not associate - /// meta to the trie, their value - /// should be trusted from an external - /// source (eg inclusion in the header - /// of a block like root), in this case - /// the implementation of this trait should - /// never be use. Then trie root is the binary_root. - /// If true the code will be use to produce - /// a root with those information and the - /// root will be hash(meta ++ binary_root), - /// so at minimal an additional round of - /// hashing. Binary root is included in meta. - /// - /// A node contaning this `meta ++ binary_root` - /// is using a prefix of length 0. - const ATTACH_TO_ROOT: bool; - - /// The buffer for this codec, this allows - /// to use fix length buffer. - type Buff: AsMut<[u8]> + AsRef<[u8]>; - /// The hash to use if we need - /// to associate meta. - type Hash: Hasher; - /// The actual meta to use. - type Meta; - /// Decode - fn decode(input: &[u8]) -> Self::Meta; - /// Encode TODO use write and stream trait? - fn encode(meta: &Self::Meta) -> Vec; - } - - /// Direct hasher as meta indicates there - /// is no meta so we can use the root directly. - impl MetaCodec for H { - const ATTACH_TO_ROOT: bool = false; - type Buff = [u8;0]; - type Hash = Self; - type Meta = (); - fn decode(_input: &[u8]) -> Self::Meta { () } - fn encode(_meta: &Self::Meta) -> Vec { Vec::new() } - } - -} - #[derive(PartialEq, Eq, Debug)] /// A binary trie with guaranties /// of content being in a fix range @@ -121,9 +56,6 @@ pub struct SequenceBinaryTree { /// global offset for index. offset: K, - /// Nb deleted values at start. - start: K, - start_depth: usize, /// Nb deleted values at end. end: K, end_depth: usize, @@ -136,25 +68,10 @@ pub struct SequenceBinaryTree { _ph: PhantomData, } -/* TODO unimplemented -pub struct SequenceBinaryTreeDB<'a, K, H: Hasher> { - tree: &'a SequenceBinaryTree, - db: &'a dyn HashDBRef, - root: &'a H::Out, -} - -pub struct SequenceBinaryTreeInMem<'a, K, NK: Ord, H: Hasher> { - tree: &'a SequenceBinaryTree, - db: &'a crate::rstd::BTreeMap, -} -*/ - impl Default for SequenceBinaryTree { fn default() -> Self { SequenceBinaryTree { offset: 0, - start: 0, - start_depth: 0, end: 0, end_depth: 0, depth: 0, @@ -173,42 +90,16 @@ fn depth(nb: usize) -> usize { ((0usize.leading_zeros() - (nb - 1).leading_zeros()) as usize) + 1*/ } -#[test] -fn test_depth() { -/* - (0, 0), - (1, 1), - (2, 2), - (3, 2), - (4, 3), - (5, 3), - (7, 3), - (8, 4), -*/ - assert_eq!(depth(0), 0); - assert_eq!(depth(1), 1); - assert_eq!(depth(2), 2); - assert_eq!(depth(3), 2); - assert_eq!(depth(4), 3); - assert_eq!(depth(7), 3); - assert_eq!(depth(8), 4); - assert_eq!(depth(9), 4); - assert_eq!(depth(u16::max_value() as usize - 1), 16); - assert_eq!(depth(u16::max_value() as usize), 16); -} - fn right_at(value: usize, index: usize) -> bool { value & (1 << index) != 0 } impl SequenceBinaryTree { - pub fn new(offset: usize, start: usize, number: usize) -> Self { - let len = start + number; + pub fn new(offset: usize, number: usize) -> Self { + let len = number; if len == 0 { SequenceBinaryTree { offset, - start, - start_depth: 0, end: 0, end_depth: 0, depth: 0, @@ -217,14 +108,11 @@ impl SequenceBinaryTree { } } else { let length = len.next_power_of_two(); - let end = length - start - number; - let start_depth = depth(start); + let end = length - number; let end_depth = depth(end); let depth = depth(length - 1); SequenceBinaryTree { offset, - start, - start_depth, end, end_depth, depth, @@ -236,7 +124,7 @@ impl SequenceBinaryTree { // TODO consider storing that fn nb_elements(&self) -> usize { - self.length - self.start - self.end + self.length - self.end } #[cfg(test)] // TODO is in implementation but unused, should be rename to resize @@ -245,7 +133,7 @@ impl SequenceBinaryTree { return; } if self.length == 0 { - *self = Self::new(self.offset, self.start, nb); + *self = Self::new(self.offset, nb); return; } while nb > self.end { @@ -386,74 +274,8 @@ impl SequenceBinaryTree { } }) } -/* TODO unimplemented - fn pop(&mut self, _nb: usize) { - unimplemented!("update max depth"); - } - - fn pop_front(&mut self, nb: usize) { - unimplemented!("update max depth"); - // TODO if start = max_depth_length / 2 -> max_depth - 1 - } - - fn max_depth_length(_end: &usize) -> usize { - // 2^x = max_depth_length - unimplemented!() - } - - fn front_depth(_index: usize) -> usize { - unimplemented!("for index between end and max len"); - } - - fn tail_depth(_index: usize) -> usize { - unimplemented!("for index between end and max len"); - } -*/ } - - -// prefix scheme, the prefix use to avoid conflict of hash in a single trie is build -// upon indexed key of the leftmost child with the depth of the prefix and then the compact encoding. -// Therefore it can be use to iterate if there is only a single state for the trie. -// -// prefix scheme: not two node with same prefix ++ hash. -// meta & root cannot happen. -// level 1 can happen: just prefix 0 or 1 -// level 2 with level 1 can happen but only on different prefix -// level 3 with level 1 -// -// no offset and the depth of , therefore compact encoding is rather suitable for it. -// We use a compact -// -// NOTE that changing trie changes depth (existing k at depth 2 moving to depth 4), therefore the scheme is rather broken -// as we cannot address the nodes anymore. -// Therefore we should prefix over the index(key) as compact. For intermediattory key it will be -// the leftmost key index. TODO make test to check no collision and write asumption that to create -// collision we need inline values of length == to hash (to target previous 2 values hash eg for 3 -// nodes trie: hash(v1,v2) = h1, v3 = h1 but this implies h1 of length of hash and this means that -// we hash the value (with inline hash of length being strictly the hash length this can be use: -// CONCLUSION even if we cannot run inline values of length of the H::Out (more should be fine as -// it implies a second round of hashing) -> can be avoided with custom encoder. -// Inline value less than size hash are a problem on the other hand: when close to size hash we can -// find collision rather easilly, but that does not work because leftmost index is 3 for v3 and 1 -// for h1 so seems rather safe. If removed from start (offset), then it is not written so safe to -// except V1 del then V2 become h(v1,v2) and then v3 = v2 does break but prefix do not move : v2 is -// still 2 and v3 is still 3 so fine to. -// Could add a value bool to the prefix or the compact encoding scheme to indicate that it is a -// terminal value -> no value are stored outside? -> seems good to iterate (same for terminal node -// with a inline value -> 4 info here : intermediate, first value, second value, both value (the -// three lasts being the same (first in fact). This lead to possible iteration by. -// For partial storage we can use same approach for a few level of intermediate (this will bound -// key size for fix prefix, then last value is reserved for compact encoding of level next which -// should really never happen). -// -// NOTE inline value does not make sense, api should only use hash, additional api could store -// access values from terminal hash. -// Prefix wise, we could store in same db with key as prefix. Also if we want to inline value, -// then the value just need to be extract from terminal hash instead. (terminal hash marker -// and value describe above is still interesting). - /// key of node is a sequence of one bit nibbles. /// This do not implement any key alignment logic, /// using it for the sequence trie should always @@ -475,89 +297,91 @@ pub trait KeyNode { } #[cfg(test)] -#[derive(Clone, Debug)] -// please do not use, only for test of (usize, K) -struct VecKeyNode(std::collections::VecDeque); -#[cfg(test)] -impl KeyNode for VecKeyNode { - fn increment_no_increase(&mut self) { - for i in (0..self.0.len()).rev() { - match self.0.get_mut(i) { - Some(v) => { - if !*v { - *v = true; - break; - } - }, - None => { - unreachable!("should only be call when guaranties to not increase depth"); - }, +mod vec_keynode { + use crate::*; + + #[derive(Clone, Debug)] + // please do not use, only for test of (usize, K) + pub(crate) struct VecKeyNode(pub(crate) std::collections::VecDeque); + + impl KeyNode for VecKeyNode { + fn increment_no_increase(&mut self) { + for i in (0..self.0.len()).rev() { + match self.0.get_mut(i) { + Some(v) => { + if !*v { + *v = true; + break; + } + }, + None => { + unreachable!("should only be call when guaranties to not increase depth"); + }, + } } } - } - fn depth(&self) -> usize { - self.0.len() - } + fn depth(&self) -> usize { + self.0.len() + } - fn nibble_at(&self, depth: usize) -> Option { - self.0.get(depth).cloned() - } - fn pop_back(&mut self) -> Option { - self.0.pop_back() - } - fn push_back(&mut self, nibble: bool) { - self.0.push_back(nibble) - } - fn pop_front(&mut self) -> Option { - self.0.pop_front() - } - fn push_front(&mut self, nibble: bool) { - self.0.push_front(nibble) - } - fn remove_at(&mut self, index: usize) { - self.0.remove(index); - } - fn starts_with(&self, other: &Self) -> bool { - // clone but it is test method only. - let mut tr = self.0.clone(); - tr.truncate(other.0.len()); - tr == other.0 - } - fn common_depth(&self, other: &Self) -> usize { - let bound = crate::rstd::cmp::min(self.0.len(), other.0.len()); - let mut depth = 0; - for i in 0..bound { - if self.0[i] == other.0[i] { - depth += 1; - } else { - break; + fn nibble_at(&self, depth: usize) -> Option { + self.0.get(depth).cloned() + } + fn pop_back(&mut self) -> Option { + self.0.pop_back() + } + fn push_back(&mut self, nibble: bool) { + self.0.push_back(nibble) + } + fn pop_front(&mut self) -> Option { + self.0.pop_front() + } + fn push_front(&mut self, nibble: bool) { + self.0.push_front(nibble) + } + fn remove_at(&mut self, index: usize) { + self.0.remove(index); + } + fn starts_with(&self, other: &Self) -> bool { + // clone but it is test method only. + let mut tr = self.0.clone(); + tr.truncate(other.0.len()); + tr == other.0 + } + fn common_depth(&self, other: &Self) -> usize { + let bound = rstd::cmp::min(self.0.len(), other.0.len()); + let mut depth = 0; + for i in 0..bound { + if self.0[i] == other.0[i] { + depth += 1; + } else { + break; + } } + depth } - depth } -} -#[cfg(test)] -impl From<(usize, usize)> for VecKeyNode { - fn from((key, depth): (usize, usize)) -> Self { - if depth == 0 { - return VecKeyNode(std::collections::VecDeque::new()); + impl From<(usize, usize)> for VecKeyNode { + fn from((key, depth): (usize, usize)) -> Self { + if depth == 0 { + return VecKeyNode(std::collections::VecDeque::new()); + } + VecKeyNode( + (1..=depth).map(|i| crate::right_at(key, depth - i)).collect() + ) } - VecKeyNode( - (1..=depth).map(|i| right_at(key, depth - i)).collect() - ) } -} -#[cfg(test)] -impl Into for VecKeyNode { - fn into(self) -> usize { - let mut result = 0; - let depth = self.depth(); - self.0.into_iter().enumerate().for_each(|(i, b)| if b { - result = result | (1 << depth - (i + 1)); - }); - result + impl Into for VecKeyNode { + fn into(self) -> usize { + let mut result = 0; + let depth = self.depth(); + self.0.into_iter().enumerate().for_each(|(i, b)| if b { + result = result | (1 << depth - (i + 1)); + }); + result + } } } @@ -676,45 +500,6 @@ impl Into for UsizeKeyNode { } } -#[test] -fn key_node_test() { - let test = |start: usize, end: bool| { - let depth = depth(start); - let mut v = VecKeyNode::from((start, depth)); - let mut u = UsizeKeyNode::from((start, depth)); - assert_eq!(v.nibble_at(start), u.nibble_at(start)); - if !end { - assert_eq!(u.push_back(true), v.push_back(true)); - } - assert_eq!(u.pop_back(), v.pop_back()); - if !end { - assert_eq!(u.push_back(false), v.push_back(false)); - } - assert_eq!(u.pop_back(), v.pop_back()); - assert_eq!(u.push_front(true), v.push_front(true)); - assert_eq!(u.pop_front(), v.pop_front()); - assert_eq!(u.push_front(false), v.push_front(false)); - assert_eq!(u.pop_front(), v.pop_front()); - if !end { - assert_eq!(start, u.into()); - assert_eq!(start, v.clone().into()); - } - assert_eq!(u.pop_back(), v.pop_back()); - let u: usize = u.into(); - assert_eq!(u, v.into()); - }; - let t: VecKeyNode = (5, 4).into(); - let t: Vec = t.0.into_iter().collect(); - assert_eq!(t, vec![false, true, false, true]); - let t: std::collections::VecDeque = [false, true, false, true].iter().cloned().collect(); - assert_eq!(5usize, VecKeyNode(t).into()); - for i in 0..17 { - test(i, false); - } - test(usize::max_value() - 1, true); -// test(usize::max_value()); -} - pub trait ProcessNode { /// Callback for an empty trie, return byte representation /// of the hash for the empty trie. @@ -979,7 +764,6 @@ pub fn trie_root(layout: &SequenceBinaryTree, input: I, cal I: Iterator, F: ProcessNode, { - debug_assert!(layout.start == 0, "unimplemented start"); let mut iter = input.into_iter().zip(layout.iter_path_node_key::(None)); debug_assert!({ let (r, s) = iter.size_hint(); @@ -1138,7 +922,7 @@ impl HasherHybrid for OrderedTrieHasher { proof: bool, buffer: &mut H::Buffer, ) -> Option { - let seq_trie = SequenceBinaryTree::new(0, 0, nb_children); + let seq_trie = SequenceBinaryTree::new(0, nb_children); let mut callback_read_proof = HashOnly::::new(buffer); let hash = if !proof { @@ -1205,11 +989,73 @@ impl Hasher for OrderedTrieHasher { #[cfg(test)] mod test { + use keccak_hasher::KeccakHasher; use super::*; - //use keccak_hasher::FixKeccakHasher; + use crate::vec_keynode::VecKeyNode; + + #[test] + fn test_depth() { + /* + (0, 0), + (1, 1), + (2, 2), + (3, 2), + (4, 3), + (5, 3), + (7, 3), + (8, 4), + */ + assert_eq!(depth(0), 0); + assert_eq!(depth(1), 1); + assert_eq!(depth(2), 2); + assert_eq!(depth(3), 2); + assert_eq!(depth(4), 3); + assert_eq!(depth(7), 3); + assert_eq!(depth(8), 4); + assert_eq!(depth(9), 4); + assert_eq!(depth(u16::max_value() as usize - 1), 16); + assert_eq!(depth(u16::max_value() as usize), 16); + } + + #[test] + fn key_node_test() { + let test = |start: usize, end: bool| { + let depth = depth(start); + let mut v = VecKeyNode::from((start, depth)); + let mut u = UsizeKeyNode::from((start, depth)); + assert_eq!(v.nibble_at(start), u.nibble_at(start)); + if !end { + assert_eq!(u.push_back(true), v.push_back(true)); + } + assert_eq!(u.pop_back(), v.pop_back()); + if !end { + assert_eq!(u.push_back(false), v.push_back(false)); + } + assert_eq!(u.pop_back(), v.pop_back()); + assert_eq!(u.push_front(true), v.push_front(true)); + assert_eq!(u.pop_front(), v.pop_front()); + assert_eq!(u.push_front(false), v.push_front(false)); + assert_eq!(u.pop_front(), v.pop_front()); + if !end { + assert_eq!(start, u.into()); + assert_eq!(start, v.clone().into()); + } + assert_eq!(u.pop_back(), v.pop_back()); + let u: usize = u.into(); + assert_eq!(u, v.into()); + }; + let t: VecKeyNode = (5, 4).into(); + let t: Vec = t.0.into_iter().collect(); + assert_eq!(t, vec![false, true, false, true]); + let t: std::collections::VecDeque = [false, true, false, true].iter().cloned().collect(); + assert_eq!(5usize, VecKeyNode(t).into()); + for i in 0..17 { + test(i, false); + } + test(usize::max_value() - 1, true); + } - //type Tree = super::SequenceBinaryTree; type Tree = super::SequenceBinaryTree; #[test] @@ -1234,7 +1080,7 @@ mod test { prev = nb; tree.push(inc); assert_eq!(tree.depth, depth); - let tree2 = Tree::new(0, 0, nb); + let tree2 = Tree::new(0, nb); assert_eq!(tree2.depth, depth); assert_eq!(tree, tree2); } @@ -1243,33 +1089,33 @@ mod test { #[test] fn test_depth_index() { // 8 trie - let tree = Tree::new(0, 0, 7); + let tree = Tree::new(0, 7); assert_eq!(tree.depth_index(3), 3); assert_eq!(tree.depth_index(4), 3); assert_eq!(tree.depth_index(6), 2); - let tree = Tree::new(0, 0, 6); + let tree = Tree::new(0, 6); assert_eq!(tree.depth_index(0), 3); assert_eq!(tree.depth_index(3), 3); assert_eq!(tree.depth_index(4), 2); assert_eq!(tree.depth_index(5), 2); - let tree = Tree::new(0, 0, 5); + let tree = Tree::new(0, 5); assert_eq!(tree.depth_index(3), 3); assert_eq!(tree.depth_index(4), 1); // 16 trie - let tree = Tree::new(0, 0, 12); + let tree = Tree::new(0, 12); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 3); assert_eq!(tree.depth_index(11), 3); - let tree = Tree::new(0, 0, 11); + let tree = Tree::new(0, 11); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 3); assert_eq!(tree.depth_index(9), 3); assert_eq!(tree.depth_index(10), 2); - let tree = Tree::new(0, 0, 10); + let tree = Tree::new(0, 10); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 2); assert_eq!(tree.depth_index(9), 2); - let tree = Tree::new(0, 0, 9); + let tree = Tree::new(0, 9); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 1); // 32 trie TODO @@ -1281,7 +1127,7 @@ mod test { // let cases = [7, 6, 5, 12, 11, 10, 9]; for nb in 0usize..16 { let mut n = 0; - let tree = Tree::new(0, 0, nb); + let tree = Tree::new(0, nb); for (i, (d, k)) in tree.iter_depth(None) .zip(tree.iter_path_node_key::(None)) .enumerate() { @@ -1420,7 +1266,7 @@ mod test { fn test_hash_only() { let result = base16_roots(); for l in 0..17 { - let tree = Tree::new(0, 0, l); + let tree = Tree::new(0, l); let mut hash_buf = ::init_buffer(); let mut callback = HashOnly::::new(&mut hash_buf); let hashes: Vec<_> = hashes(l); @@ -1433,7 +1279,7 @@ mod test { fn test_one_element_proof() { let result = base16_roots(); for l in 0..17 { - let tree = Tree::new(0, 0, l); + let tree = Tree::new(0, l); let mut hash_buf = ::init_buffer(); let mut hash_buf2 = ::init_buffer(); let mut hash_buf3 = ::init_buffer(); @@ -1487,7 +1333,7 @@ mod test { for (l, ps) in tests.iter() { let l = *l; let ps = *ps; - let tree = Tree::new(0, 0, l); + let tree = Tree::new(0, l); let mut hash_buf = ::init_buffer(); let mut hash_buf2 = ::init_buffer(); let mut hash_buf3 = ::init_buffer(); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 1de82386..b0e1a8b5 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -588,7 +588,7 @@ pub fn binary_additional_hashes( hash_buf: &mut H::Buffer, ) -> Vec { let nb_children = children.iter().filter(|v| v.is_some()).count(); - let tree = SequenceBinaryTree::new(0, 0, nb_children); + let tree = SequenceBinaryTree::new(0, nb_children); let to_prove = children.iter().zip(in_proof_children.iter()) .filter_map(|(o_child, in_proof)| o_child.as_ref().map(|_| *in_proof)) From 1c27a54b4c7ff04481de325c0d4ac423b719c605 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 12 May 2020 18:02:41 +0200 Subject: [PATCH 36/73] allow using two different hasher with same output size in hybrid trie --- hash-db/src/lib.rs | 9 +++++---- memory-db/src/lib.rs | 4 ++-- ordered-trie/src/lib.rs | 15 ++++++++------- test-support/reference-trie/src/lib.rs | 2 +- trie-db/src/iter_build.rs | 24 ++++++++++++------------ trie-db/src/node_codec.rs | 10 +++++----- trie-db/src/proof/verify.rs | 4 ++-- trie-db/src/trie_codec.rs | 2 +- trie-db/src/triedbmut.rs | 6 +++--- 9 files changed, 39 insertions(+), 37 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 256c5532..4b0dd6cf 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -229,7 +229,7 @@ impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { /// Same as HashDB but can modify the value upon storage, and apply /// `HasherHybrid`. -pub trait HashDBHybrid: Send + Sync + HashDB { +pub trait HashDBHybrid: Send + Sync + HashDB { /// `HashDB` is often use to load content from encoded node. /// This will not preserve insertion done through `insert_branch_hybrid` calls /// and break the proof. @@ -260,22 +260,23 @@ pub trait HashDBHybrid: Send + Sync + HashDB { children: I, additional_hashes: I2, proof: bool, - buffer: &mut H::Buffer, + buffer: &mut ::Buffer, ) -> H::Out; } pub trait HasherHybrid: BinaryHasher { + type InnerHasher: BinaryHasher; /// Alternate hash with hybrid proof allowed fn hash_hybrid< I: Iterator::Out>>, - I2: Iterator::Out>, + I2: Iterator::Out>, >( x: &[u8], nb_children: usize, children: I, additional_hashes: I2, proof: bool, - buffer: &mut Self::Buffer, + buffer: &mut ::Buffer, ) -> Option; } diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 813e2efb..7263b5a7 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -20,7 +20,7 @@ extern crate alloc; use hash_db::{HashDB, HashDBRef, PlainDB, PlainDBRef, Hasher as KeyHasher, - AsHashDB, AsPlainDB, Prefix, HashDBHybrid, HasherHybrid}; + AsHashDB, AsPlainDB, Prefix, HashDBHybrid, HasherHybrid, BinaryHasher}; use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; #[cfg(feature = "deprecated")] #[cfg(feature = "std")] @@ -639,7 +639,7 @@ where children: I, additional_hashes: I2, proof: bool, - buff: &mut H::Buffer, + buff: &mut ::Buffer, ) -> H::Out { if T::from(value) == self.null_node_data { return self.hashed_null_node.clone(); diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 9e1ebac9..b7a14da5 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -906,11 +906,12 @@ pub fn trie_root_from_proof( } } -#[repr(transparent)] /// Hasher hybrid using an ordered trie -pub struct OrderedTrieHasher(H); +pub struct OrderedTrieHasher(PhantomData<(H, HH)>); + +impl> HasherHybrid for OrderedTrieHasher { + type InnerHasher = HH; -impl HasherHybrid for OrderedTrieHasher { fn hash_hybrid< I: Iterator::Out>>, I2: Iterator::Out>, @@ -920,11 +921,11 @@ impl HasherHybrid for OrderedTrieHasher { children: I, additional_hashes: I2, proof: bool, - buffer: &mut H::Buffer, + buffer: &mut HH::Buffer, ) -> Option { let seq_trie = SequenceBinaryTree::new(0, nb_children); - let mut callback_read_proof = HashOnly::::new(buffer); + let mut callback_read_proof = HashOnly::::new(buffer); let hash = if !proof { // full node let iter = children.filter_map(|v| v); // TODO assert same number as count @@ -959,7 +960,7 @@ impl HasherHybrid for OrderedTrieHasher { } } -impl BinaryHasher for OrderedTrieHasher { +impl BinaryHasher for OrderedTrieHasher { const NULL_HASH: &'static [u8] = ::NULL_HASH; type Buffer = ::Buffer; fn init_buffer() -> Self::Buffer { @@ -976,7 +977,7 @@ impl BinaryHasher for OrderedTrieHasher { } } -impl Hasher for OrderedTrieHasher { +impl Hasher for OrderedTrieHasher { type Out = ::Out; type StdHasher = ::StdHasher; const LENGTH: usize = ::LENGTH; diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 89ab4083..6bbd9b0b 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -48,7 +48,7 @@ pub mod node { } /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. -pub type RefHasher = ordered_trie::OrderedTrieHasher; +pub type RefHasher = ordered_trie::OrderedTrieHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; /// Trie layout using extension nodes. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 03d7e4ba..084fd6a9 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -395,16 +395,16 @@ impl<'a, H, HO, V, DB> TrieBuilder<'a, H, HO, V, DB> { /// Get trie root and insert visited node in a hash_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). -pub struct TrieBuilderHybrid<'a, H: BinaryHasher, HO, V, DB> { +pub struct TrieBuilderHybrid<'a, H: HasherHybrid, HO, V, DB> { db: &'a mut DB, pub root: Option, - buffer: H::Buffer, + buffer: ::Buffer, _ph: PhantomData<(H, V)>, } -impl<'a, H: BinaryHasher, HO, V, DB> TrieBuilderHybrid<'a, H, HO, V, DB> { +impl<'a, H: HasherHybrid, HO, V, DB> TrieBuilderHybrid<'a, H, HO, V, DB> { pub fn new(db: &'a mut DB) -> Self { - TrieBuilderHybrid { db, root: None, buffer: H::init_buffer(), _ph: PhantomData } + TrieBuilderHybrid { db, root: None, buffer: H::InnerHasher::init_buffer(), _ph: PhantomData } } } @@ -516,15 +516,15 @@ impl ProcessEncodedNode<::Out> for TrieRoot { +pub struct TrieRootHybrid { /// The resulting root. pub root: Option, - buffer: H::Buffer, + buffer: ::Buffer, } -impl Default for TrieRootHybrid { +impl Default for TrieRootHybrid { fn default() -> Self { - TrieRootHybrid { root: None, buffer: H::init_buffer() } + TrieRootHybrid { root: None, buffer: H::InnerHasher::init_buffer() } } } @@ -584,15 +584,15 @@ impl Default for TrieRootUnhashed { } /// Get the trie root node encoding. -pub struct TrieRootUnhashedHybrid { +pub struct TrieRootUnhashedHybrid { /// The resulting encoded root. pub root: Option>, - buffer: H::Buffer, + buffer: ::Buffer, } -impl Default for TrieRootUnhashedHybrid { +impl Default for TrieRootUnhashedHybrid { fn default() -> Self { - TrieRootUnhashedHybrid { root: None, buffer: H::init_buffer() } + TrieRootUnhashedHybrid { root: None, buffer: H::InnerHasher::init_buffer() } } } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 780efcb7..6beaebb5 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -205,7 +205,7 @@ impl ChildProofHeader { use hash_db::{HashDB, Prefix, HashDBRef, Hasher, HashDBHybrid}; -pub trait HashDBHybridDyn: Send + Sync + HashDB { +pub trait HashDBHybridDyn: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. @@ -218,7 +218,7 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { value: &[u8], children: &[Option>], common: ChildProofHeader, - buffer: &mut H::Buffer, + buffer: &mut ::Buffer, ) -> H::Out; } @@ -229,7 +229,7 @@ impl> HashDBHybridDyn for C { value: &[u8], children: &[Option>], common: ChildProofHeader, - buffer: &mut H::Buffer, + buffer: &mut ::Buffer, ) -> H::Out { // TODO factor this with iter_build (just use the trait) also use in adapter from codec @@ -394,8 +394,8 @@ pub fn hybrid_hash_node_adapter, H dest.as_mut()[..range.len()].copy_from_slice(&encoded_node[range]); dest })); - let mut buf = Hasher::init_buffer(); - Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( + let mut buf = ::InnerHasher::init_buffer(); + Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( common.header(encoded_node), nb_children, children, diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index b808286b..779637f0 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -458,7 +458,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve L::HYBRID_HASH, )?; let mut hybrid_buf = if L::HYBRID_HASH { - Some(L::Hash::init_buffer()) + Some(::InnerHasher::init_buffer()) } else { None }; loop { // Insert omitted value. @@ -506,7 +506,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } }); - if let Some(h) = OrderedTrieHasher::::hash_hybrid( + if let Some(h) = OrderedTrieHasher::::InnerHasher>::hash_hybrid( &common.header(node_data.as_slice())[..], nb_children, children, diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index b0e1a8b5..e1b1111d 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -509,7 +509,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) }; let mut hybrid_buf = if L::HYBRID_HASH { - Some(L::Hash::init_buffer()) + Some(::InnerHasher::init_buffer()) } else { None }; loop { if !last_entry.advance_child_index()? { diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index d1654d8e..f0a2817b 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -20,7 +20,7 @@ use super::lookup::Lookup; use super::node::{NodeHandle as EncodedNodeHandle, Node as EncodedNode, decode_hash}; use crate::node_codec::HashDBHybridDyn; -use hash_db::{Hasher, Prefix, EMPTY_PREFIX, BinaryHasher}; +use hash_db::{Hasher, Prefix, EMPTY_PREFIX, BinaryHasher, HasherHybrid}; use hashbrown::HashSet; use crate::node_codec::{NodeCodec, NodeCodecHybrid, ChildProofHeader}; @@ -449,7 +449,7 @@ where /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, - hybrid_hash_buffer: Option<::Buffer>, + hybrid_hash_buffer: Option<<::InnerHasher as BinaryHasher>::Buffer>, } impl<'a, L> TrieDBMut<'a, L> @@ -486,7 +486,7 @@ where fn hybrid_hash_buffer_lazy_init(&mut self) { if self.hybrid_hash_buffer.is_none() { - self.hybrid_hash_buffer = Some(L::Hash::init_buffer()) + self.hybrid_hash_buffer = Some(::InnerHasher::init_buffer()) } } From 9411f44fa9f7a68d88d3b2b024c78a7868e8ab9f Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 13 May 2020 20:08:27 +0200 Subject: [PATCH 37/73] - remove inline node from compact encoded when not in value to prove - fix usage of two different hasher for compact proof without value --- test-support/reference-trie/src/lib.rs | 43 +++++---- trie-db/fuzz/Cargo.toml | 4 + trie-db/fuzz/fuzz_targets/trie_codec_proof.rs | 8 ++ trie-db/fuzz/src/lib.rs | 90 ++++++++++++++++++- trie-db/src/node.rs | 8 ++ trie-db/src/node_codec.rs | 13 ++- trie-db/src/proof/generate.rs | 31 ++++--- trie-db/src/trie_codec.rs | 33 ++++++- 8 files changed, 193 insertions(+), 37 deletions(-) create mode 100644 trie-db/fuzz/fuzz_targets/trie_codec_proof.rs diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 6bbd9b0b..ecbaf5b0 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -30,6 +30,7 @@ use trie_db::{ TrieRootHybrid, Partial, HasherHybrid, + BinaryHasher, ChildProofHeader, HashesPlan, binary_additional_hashes, @@ -49,6 +50,7 @@ pub mod node { /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. pub type RefHasher = ordered_trie::OrderedTrieHasher; +//pub type RefHasher = ordered_trie::OrderedTrieHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; /// Trie layout using extension nodes. @@ -515,7 +517,7 @@ impl Decode for NodeHeader { impl NodeHeaderNoExt { fn is_branch(first_byte: u8) -> bool { let first_byte = first_byte & (0b11 << 6); - first_byte == BRANCH_WITHOUT_MASK_NO_EXT + first_byte == BRANCH_WITHOUT_MASK_NO_EXT || first_byte == BRANCH_WITH_MASK_NO_EXT } } @@ -853,12 +855,13 @@ impl NodeCodecHybrid for ReferenceNodeCodec { unreachable!() } - fn encode_compact_proof( + fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], hash_buf: &mut BH::Buffer, + in_proof: &[bool], ) -> Vec { - encode_proof_internal::(hash_proof_header, children, hash_buf) + encode_proof_internal::(hash_proof_header, children, hash_buf, in_proof) } fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)> { @@ -999,37 +1002,40 @@ fn decode_plan_compact_proof_internal( Ok((node, hashes_plan)) } -fn encode_proof_internal( +fn encode_proof_internal( mut result: Vec, children: &[Option>], hash_buf: &mut H::Buffer, + in_proof: &[bool], ) -> Vec { - let mut in_proof_children = [false; nibble_ops::NIBBLE_LENGTH]; let bitmap_start = result.len(); result.push(0u8); result.push(0u8); - // Write all inline nodes. + // Write all inline nodes, that need to be proved. for (ix, child) in children.iter().enumerate() { if let Some(ChildReference::Inline(h, nb)) = child.borrow() { if *nb > 0 { - // TODO do not write inline of null size, these are defined - // in the bitmap and from the algos. - debug_assert!(*nb < 128); - result.push(*nb as u8); - result.push(ix as u8); - result.extend_from_slice(&h.as_ref()[..*nb]); + if in_proof[ix] { + // TODO do not write inline of null size, these are defined + // in the bitmap and from the algos. + debug_assert!(*nb < 128); + result.push(*nb as u8); + result.push(ix as u8); + result.extend_from_slice(&h.as_ref()[..*nb]); + } + } else { + debug_assert!(in_proof[ix]); } - in_proof_children[ix] = true; } } // We write a bitmap containing all children node that are included in the binary // child proof construction. // In practice, that is inline values and ommited compacted values). - Bitmap::encode(in_proof_children.iter().map(|b| *b), &mut result[bitmap_start..]); + Bitmap::encode(in_proof.iter().map(|b| *b), &mut result[bitmap_start..]); let additional_hashes = binary_additional_hashes::( &children[..], - &in_proof_children[..], + &in_proof[..], hash_buf, ); result.push((additional_hashes.len() as u8) | 128); // first bit at one indicates we are on additional hashes @@ -1309,12 +1315,13 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { ).0 } - fn encode_compact_proof( + fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], hash_buf: &mut BH::Buffer, + in_proof: &[bool], ) -> Vec { - encode_proof_internal::(hash_proof_header, children, hash_buf) + encode_proof_internal::(hash_proof_header, children, hash_buf, in_proof) } fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)> { @@ -1510,7 +1517,7 @@ pub fn compare_implementations_no_extension( } t.root().clone() }; - + if root != root_new { { let db : &dyn hash_db::HashDB<_, _> = &memdb; diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index 434d684c..bff842f5 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -61,6 +61,10 @@ path = "fuzz_targets/seek_iter.rs" name = "trie_proof_valid" path = "fuzz_targets/trie_proof_valid.rs" +[[bin]] +name = "trie_codec_proof" +path = "fuzz_targets/trie_codec_proof.rs" + [[bin]] name = "trie_proof_invalid" path = "fuzz_targets/trie_proof_invalid.rs" diff --git a/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs b/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs new file mode 100644 index 00000000..7e28ec89 --- /dev/null +++ b/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use trie_db_fuzz::fuzz_that_trie_codec_proofs; + +fuzz_target!(|data: &[u8]| { + fuzz_that_trie_codec_proofs(data); +}); diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 0745da4d..374f803d 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -293,6 +293,25 @@ pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) { assert!(verify_proof::(&root, &proof, items.iter()).is_ok()); } +pub fn fuzz_that_trie_codec_proofs(input: &[u8]) { + let mut data = fuzz_to_data(input); + // Split data into 3 parts: + // - the first 1/3 is added to the trie and not included in the proof + // - the second 1/3 is added to the trie and included in the proof + // - the last 1/3 is not added to the trie and the proof proves non-inclusion of them + let mut keys = data[(data.len() / 3)..] + .iter() + .map(|(key, _)| key.clone()) + .collect::>(); + data.truncate(data.len() * 2 / 3); + + let data = data_sorted_unique(data); + keys.sort(); + keys.dedup(); + + test_trie_codec_proof::(data, keys); +} + pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) { if input.len() < 4 { return; @@ -364,9 +383,74 @@ fn test_generate_proof( (root, proof, items) } -/* +fn test_trie_codec_proof( + entries: Vec<(Vec, Vec)>, + keys: Vec>, +) +{ + use hash_db::{HashDB, EMPTY_PREFIX}; + use reference_trie::{ + Recorder, + encode_compact, decode_compact, + }; + + // Populate DB with full trie from entries. + let (db, root) = { + let mut db = , _>>::default(); + let mut root = Default::default(); + { + let mut trie = >::new(&mut db, &mut root); + for (key, value) in entries { + trie.insert(&key, &value).unwrap(); + } + } + (db, root) + }; + let expected_root = root; + // Lookup items in trie while recording traversed nodes. + let mut recorder = Recorder::new(); + let items = { + let mut items = Vec::with_capacity(keys.len()); + let trie = >::new(&db, &root).unwrap(); + for key in keys { + let value = trie.get_with(key.as_slice(), &mut recorder).unwrap(); + items.push((key, value)); + } + items + }; + + // Populate a partial trie DB with recorded nodes. + let mut partial_db = , _>>::default(); + for record in recorder.drain() { + partial_db.emplace(record.hash, EMPTY_PREFIX, record.data); + } + + // Compactly encode the partial trie DB. + let compact_trie = { + let trie = >::new(&partial_db, &root).unwrap(); + encode_compact::(&trie).unwrap() + }; + + let expected_used = compact_trie.len(); + // Reconstruct the partial DB from the compact encoding. + let mut db = , _>>::default(); + let (root, used) = decode_compact::(&mut db, &compact_trie).unwrap(); + assert_eq!(root, expected_root); + assert_eq!(used, expected_used); + + // Check that lookups for all items succeed. + let trie = >::new(&db, &root).unwrap(); + for (key, expected_value) in items { + assert_eq!(trie.get(key.as_slice()).unwrap(), expected_value); + } +} + + + #[test] fn debug_error() { - let input = [0x40,0x0,0xe6,0xff,0xff,0x40,0x0,0xe6,0xff,0xff,0x2b]; + let input = [0x0,0x0,0x0,0xcd,0xf8,0xff,0xff,0xff,0x0,0xcd,0x2]; +// let input = [0x40,0x0,0xe6,0xff,0xff,0x40,0x0,0xe6,0xff,0xff,0x2b]; fuzz_that_verify_accepts_valid_proofs(&input[..]); -}*/ + fuzz_that_trie_codec_proofs(&input[..]); +} diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 46be0038..66f95e01 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -94,6 +94,14 @@ impl NodeHandlePlan { NodeHandlePlan::Inline(range) => range.clone(), } } + + /// Check if it is an inline node. + pub fn is_inline(&self) -> bool { + match self { + NodeHandlePlan::Hash(_) => false, + NodeHandlePlan::Inline(_) => true, + } + } } /// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice. The diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 6beaebb5..2410c809 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -106,11 +106,11 @@ pub trait NodeCodecHybrid: NodeCodec { /// are not use by the proof, in this case they are stored as an inline /// zero length child. /// - /// Accompanying resulting node, is also a bitmap of children included in + /// With resulting node is attached a bitmap of children included in /// the proof calculation and a sequence of additianal node for proof /// verification. - /// In practice this contains inline node (inline nodes are always added - /// to the proof) and ommitted children hash (compacted hash). + /// In practice this contains inline node that are in the proof and + /// and ommitted children hash (compacted hash). fn decode_compact_proof(data: &[u8]) -> Result<( Node, Option<(Bitmap, HashesIter)>, @@ -127,14 +127,19 @@ pub trait NodeCodecHybrid: NodeCodec { /// `ChildProofHeader` call, or directly by `branch_node_for_hash`. /// - `children`: contains all children, with compact (ommited children) defined as /// a null length inline node. + /// - `hash_buff`: technical buffer for the hasher of the second level proof. + /// - `in_proof`: position of nodes that are included in proof, this includes + /// hash nodes (can be deducted from unsized children) and inline nodes. /// The children to be include in the proof are therefore the compacted one and the /// inline nodes only. /// The other children value are needed because they can be included into the additional /// hash, and are required for intermediate hash calculation. - fn encode_compact_proof( + fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], + // TODO EMCH siwtch position on 2 last params hash_buf: &mut H::Buffer, + in_proof: &[bool], ) -> Vec; /// Does the encoded content need a hybrid proof. diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 0a2dece8..280c8620 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -18,7 +18,7 @@ use crate::rstd::{ boxed::Box, convert::TryInto, marker::PhantomData, ops::Range, vec, vec::Vec, }; -use hash_db::{Hasher, HasherHybrid}; +use hash_db::{Hasher, BinaryHasher, HasherHybrid}; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, NibbleSlice, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode}, NodeCodec, Recorder, @@ -42,6 +42,9 @@ struct StackEntry<'a, C: NodeCodec, H> { /// The index into the proof vector that the encoding of this entry should be placed at. output_index: Option, is_inline: bool, + /// Flags indicating whether each child is into the proof, this is all `omit_children` + /// child plus inline nodes. + in_proof_children: [bool; NIBBLE_LENGTH], _marker: PhantomData<(C, H)>, } @@ -75,6 +78,7 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> children: vec![None; children_len], output_index, is_inline, + in_proof_children: [false; NIBBLE_LENGTH], _marker: PhantomData::default(), }) } @@ -83,7 +87,7 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> fn encode_node( mut self, hybrid: bool, - hash_buf: &mut H::Buffer, + hash_buf: &mut ::Buffer, ) -> TrieResult, C::HashOut, C::Error> { let node_data = self.node.data(); Ok(match self.node.node_plan() { @@ -120,10 +124,11 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> self.children.iter(), value_with_omission(node_data, value, self.omit_value), ); - C::encode_compact_proof::( + C::encode_compact_proof::( hash_proof_header, &self.children[..], hash_buf, + &self.in_proof_children[..], ) } else { C::branch_node( @@ -147,10 +152,11 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> self.children.iter(), value_with_omission(node_data, value, self.omit_value), ); - C::encode_compact_proof::( + C::encode_compact_proof::( hash_proof_header, &self.children[..], hash_buf, + &self.in_proof_children[..], ) } else { C::branch_node_nibbled( @@ -209,7 +215,7 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> set_child is called when the only child is popped from the stack; \ child_index is 0 before child is pushed to the stack; qed" ); - Some(Self::replacement_child_ref(encoded_child, child)) + (false, Some(Self::replacement_child_ref(encoded_child, child))) } NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { assert!( @@ -218,12 +224,15 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> set_child is called when the only child is popped from the stack; \ child_index is (trie: &T, keys: I) .collect::>(); keys.sort(); keys.dedup(); - let mut hash_buf = ::init_buffer(); + let mut hash_buf = <::InnerHasher as hash_db::BinaryHasher>::init_buffer(); let hash_buf = &mut hash_buf; @@ -542,7 +551,7 @@ fn unwind_stack( proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, hybrid: bool, - hash_buf: &mut H::Buffer, + hash_buf: &mut ::Buffer, ) -> TrieResult<(), C::HashOut, C::Error> where H: HasherHybrid, diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index e1b1111d..65c1bf47 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -46,6 +46,9 @@ struct EncoderStackEntry { child_index: usize, /// Flags indicating whether each child is omitted in the encoded node. omit_children: [bool; NIBBLE_LENGTH], + /// Flags indicating whether each child is into the proof, this is all `omit_children` + /// child plus inline nodes. + in_proof_children: [bool; NIBBLE_LENGTH], /// The encoding of the subtrie nodes rooted at this entry, which is built up in /// `encode_compact`. output_index: usize, @@ -96,6 +99,7 @@ impl EncoderStackEntry { Ok(()) } + /// Generates the encoding of the subtrie rooted at this entry. fn encode_node( &self, @@ -128,6 +132,7 @@ impl EncoderStackEntry { hash_proof_header, &children[..], hash_buf, + &self.in_proof_children[..], ) } else { C::branch_node( @@ -150,6 +155,7 @@ impl EncoderStackEntry { hash_proof_header, &children[..], hash_buf, + &self.in_proof_children, ) } else { C::branch_node_nibbled( @@ -232,6 +238,26 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE // Skip inline nodes, as they cannot contain hash references to other nodes by // assumption. if node_hash.is_none() { + // still need to register them when in proof path for hybrid. + if L::HYBRID_HASH { + loop { + if let Some(last_entry) = stack.last_mut() { + if prefix.starts_with(&last_entry.prefix) { + let child_index = prefix.at(prefix.len() - 1) as usize; + last_entry.in_proof_children[child_index] = true; + break; + } else { + // unstack as in not inline case + if let Some(last_entry) = stack.pop() { + output[last_entry.output_index] = last_entry.encode_node::( + L::HYBRID_HASH, + hash_buf, + )?; + } + } + } else { break; } + } + } continue; } @@ -249,6 +275,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE TrieDBNodeIterator or this function" ); last_entry.omit_children[last_entry.child_index] = true; + last_entry.in_proof_children[last_entry.child_index] = true; last_entry.child_index += 1; stack.push(last_entry); break; @@ -265,6 +292,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE node, child_index: 0, omit_children: [false; NIBBLE_LENGTH], + in_proof_children: [false; NIBBLE_LENGTH], output_index: output.len(), _marker: PhantomData::default(), }); @@ -582,7 +610,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) /// indicates if it is included in the proof (inline node or /// compacted node). /// - `hash_buf` a buffer of the right size to compute the hash. -pub fn binary_additional_hashes( +pub fn binary_additional_hashes( children: &[Option>], in_proof_children: &[bool], hash_buf: &mut H::Buffer, @@ -648,6 +676,7 @@ mod tests { let items = { let mut items = Vec::with_capacity(keys.len()); let trie = >::new(&db, &root).unwrap(); + println!("{:?}", trie); for key in keys { let value = trie.get_with(key, &mut recorder).unwrap(); items.push((key, value)); @@ -664,6 +693,7 @@ mod tests { // Compactly encode the partial trie DB. let compact_trie = { let trie = >::new(&partial_db, &root).unwrap(); + println!("{:?}", trie); encode_compact::(&trie).unwrap() }; @@ -684,6 +714,7 @@ mod tests { // Check that lookups for all items succeed. let trie = >::new(&db, &root).unwrap(); + println!("{:?}", trie); for (key, expected_value) in items { assert_eq!(trie.get(key).unwrap(), expected_value); } From f196e7d4f80b86d1762ce00375e6b0f3a176b650 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 10:53:43 +0200 Subject: [PATCH 38/73] actually remove the deps from previous refactor --- memory-db/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/memory-db/Cargo.toml b/memory-db/Cargo.toml index d8489577..445df434 100644 --- a/memory-db/Cargo.toml +++ b/memory-db/Cargo.toml @@ -11,7 +11,6 @@ edition = "2018" heapsize = { version = "0.4", optional = true } parity-util-mem = { version = "0.6", default-features = false } hash-db = { path = "../hash-db", default-features = false, version = "0.15.2"} -ordered-trie = { path = "../ordered-trie", default-features = false, version = "0.19.2"} # TODO could probably remove (work in hashdb) hashbrown = { version = "0.6.3", default-features = false, features = [ "ahash" ] } # There's a compilation error with ahash-0.2.17, which is permitted by the 0.2.11 constraint in hashbrown. ahash = "0.2.18" @@ -24,7 +23,6 @@ criterion = "0.2.8" default = ["std"] std = [ "hash-db/std", - "ordered-trie/std", "parity-util-mem/std", ] deprecated = [ "heapsize" ] From 91c48d8fd50fc4a37e4f357733104bd1c2a7f137 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 11:45:47 +0200 Subject: [PATCH 39/73] return none on wrong encoded value for insert hybrid --- hash-db/src/lib.rs | 3 ++- memory-db/src/lib.rs | 19 +++++++------------ test-support/reference-trie/src/lib.rs | 8 ++++++++ trie-db/src/iter_build.rs | 4 ++-- trie-db/src/node_codec.rs | 9 +++++++-- trie-db/src/trie_codec.rs | 11 +++++++++-- trie-db/src/triedbmut.rs | 4 ++-- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 4b0dd6cf..91303cc4 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -248,6 +248,7 @@ pub trait HashDBHybrid: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. + /// This function can fail on condition from insert_hybrid, in this case we return `None`. fn insert_branch_hybrid< I: Iterator>, I2: Iterator, @@ -261,7 +262,7 @@ pub trait HashDBHybrid: Send + Sync + HashDB { additional_hashes: I2, proof: bool, buffer: &mut ::Buffer, - ) -> H::Out; + ) -> Option; } pub trait HasherHybrid: BinaryHasher { diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 7263b5a7..d564d496 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -626,7 +626,7 @@ where false } } - + fn insert_branch_hybrid< I: Iterator>, I2: Iterator, @@ -640,27 +640,22 @@ where additional_hashes: I2, proof: bool, buff: &mut ::Buffer, - ) -> H::Out { + ) -> Option { if T::from(value) == self.null_node_data { - return self.hashed_null_node.clone(); + return Some(self.hashed_null_node.clone()); } - let key = if let Some(key) = H::hash_hybrid( + H::hash_hybrid( child_proof_header, nb_children, children, additional_hashes, proof, buff, - ) { + ).map(|key| { + HashDB::emplace(self, key, prefix, value.into()); key - } else { - // invalid proof TODO handle error properly - return self.hashed_null_node.clone(); - }; - - HashDB::emplace(self, key, prefix, value.into()); - key + }) } } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index ecbaf5b0..23228ed3 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -878,6 +878,10 @@ impl NodeCodecHybrid for ReferenceNodeCodec { } None } + + fn codec_error(desc: &'static str) -> Self::Error { + desc.into() + } } impl ReferenceNodeCodec { @@ -1338,6 +1342,10 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { } None } + + fn codec_error(desc: &'static str) -> Self::Error { + desc.into() + } } /// Compare trie builder and in memory trie. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 084fd6a9..e7490edf 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -467,7 +467,7 @@ impl<'a, H: HasherHybrid, V, DB: HashDBHybrid> ProcessEncodedNode< ProcessEncodedNode<::Out> for TrieRootHybrid< EmptyIter::default(), false, &mut self.buffer, - ).expect("only proof fails: TODO two primitives") + ).expect("This node encoding is trusted") } else { ::hash(&encoded_node[..]) }; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 2410c809..458b472a 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -184,6 +184,11 @@ pub trait NodeCodecHybrid: NodeCodec { children: impl Iterator>>>, value: Option<&[u8]>, ) -> Vec; + + /// Return a error from a static description. + /// Depending on implementation it is fine drop the description + /// and act as a default error semantic. + fn codec_error(desc: &'static str) -> Self::Error; } /// Information to fetch bytes that needs to be include when calculating a node hash. @@ -224,7 +229,7 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { children: &[Option>], common: ChildProofHeader, buffer: &mut ::Buffer, - ) -> H::Out; + ) -> Option; } impl> HashDBHybridDyn for C { @@ -235,7 +240,7 @@ impl> HashDBHybridDyn for C { children: &[Option>], common: ChildProofHeader, buffer: &mut ::Buffer, - ) -> H::Out { + ) -> Option { // TODO factor this with iter_build (just use the trait) also use in adapter from codec let nb_children = children.iter().filter(|v| v.is_some()).count(); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 65c1bf47..8082fcfc 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -575,7 +575,7 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) None } }); - db.insert_branch_hybrid( + if let Some(hash) = db.insert_branch_hybrid( prefix.as_prefix(), &node_data[..], common.header(&node_data[..]), @@ -584,7 +584,14 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) additional_hashes, true, hybrid_buf.as_mut().expect("Initialized for hybrid above"), - ) + ) { + hash + } else { + return Err(Box::new(TrieError::DecoderError( + >::default(), + L::Codec::codec_error("Invalid encoding on hybrid hash calculation"), + ))) + } } else { db.insert(prefix.as_prefix(), node_data.as_ref()) }; diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index f0a2817b..560963cd 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1488,7 +1488,7 @@ where &children[..], no_node, self.hybrid_hash_buffer.as_mut().expect("Lazy init above"), - ); + ).expect("Trusted encoded value (was change earlier)") } else { *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root[..]); } @@ -1548,7 +1548,7 @@ where &children[..], no_node, self.hybrid_hash_buffer.as_mut().expect("Lazy init above"), - ) + ).expect("Trusted encoded value (was change earlier)") } else { self.db.insert(prefix.as_prefix(), &encoded[..]) }; From 22a4ef557e2cc9647976dd0dc9f2ccdac99235b1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 15:47:03 +0200 Subject: [PATCH 40/73] correct db traits --- hash-db/src/lib.rs | 30 +++++++++++++--- memory-db/src/lib.rs | 30 ++++++++++++++-- ordered-trie/src/lib.rs | 68 ++++++++++++++++++++++--------------- trie-db/src/iter_build.rs | 12 ++----- trie-db/src/node_codec.rs | 15 +++----- trie-db/src/proof/verify.rs | 3 +- trie-db/src/trie_codec.rs | 3 +- trie-db/src/triedbmut.rs | 4 +-- 8 files changed, 106 insertions(+), 59 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 91303cc4..f53531d5 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -248,9 +248,22 @@ pub trait HashDBHybrid: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. - /// This function can fail on condition from insert_hybrid, in this case we return `None`. fn insert_branch_hybrid< I: Iterator>, + >( + &mut self, + prefix: Prefix, + value: &[u8], + no_child_value: &[u8], + nb_children: usize, + children: I, + buffer: &mut ::Buffer, + ) -> H::Out; + + /// Insert with data from a proof. + /// As a result, this function can fail. + fn insert_branch_hybrid_proof< + I: Iterator>, I2: Iterator, >( &mut self, @@ -260,7 +273,6 @@ pub trait HashDBHybrid: Send + Sync + HashDB { nb_children: usize, children: I, additional_hashes: I2, - proof: bool, buffer: &mut ::Buffer, ) -> Option; } @@ -268,16 +280,26 @@ pub trait HashDBHybrid: Send + Sync + HashDB { pub trait HasherHybrid: BinaryHasher { type InnerHasher: BinaryHasher; - /// Alternate hash with hybrid proof allowed + /// Alternate hash with hybrid hashing allowed. fn hash_hybrid< I: Iterator::Out>>, + >( + x: &[u8], + nb_children: usize, + children: I, + buffer: &mut ::Buffer, + ) -> Self::Out; + + /// Calculate hash from a proof, this can fail. + fn hash_hybrid_proof< + I: Iterator::Out>>, I2: Iterator::Out>, >( x: &[u8], nb_children: usize, children: I, additional_hashes: I2, - proof: bool, buffer: &mut ::Buffer, ) -> Option; + } diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index d564d496..05287b35 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -629,6 +629,31 @@ where fn insert_branch_hybrid< I: Iterator>, + > ( + &mut self, + prefix: Prefix, + value: &[u8], + child_proof_header: &[u8], + nb_children: usize, + children: I, + buff: &mut ::Buffer, + ) -> H::Out { + if T::from(value) == self.null_node_data { + return self.hashed_null_node.clone(); + } + + let key = H::hash_hybrid( + child_proof_header, + nb_children, + children, + buff, + ); + HashDB::emplace(self, key, prefix, value.into()); + key + } + + fn insert_branch_hybrid_proof< + I: Iterator>, I2: Iterator, > ( &mut self, @@ -638,25 +663,24 @@ where nb_children: usize, children: I, additional_hashes: I2, - proof: bool, buff: &mut ::Buffer, ) -> Option { if T::from(value) == self.null_node_data { return Some(self.hashed_null_node.clone()); } - H::hash_hybrid( + H::hash_hybrid_proof( child_proof_header, nb_children, children, additional_hashes, - proof, buff, ).map(|key| { HashDB::emplace(self, key, prefix, value.into()); key }) } + } impl HashDBRef for MemoryDB diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index b7a14da5..da30d6f3 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -912,7 +912,7 @@ pub struct OrderedTrieHasher(PhantomData<(H, HH)>); impl> HasherHybrid for OrderedTrieHasher { type InnerHasher = HH; - fn hash_hybrid< + fn hash_hybrid_proof< I: Iterator::Out>>, I2: Iterator::Out>, >( @@ -920,44 +920,58 @@ impl> HasherHybrid for OrderedTr nb_children: usize, children: I, additional_hashes: I2, - proof: bool, buffer: &mut HH::Buffer, ) -> Option { let seq_trie = SequenceBinaryTree::new(0, nb_children); let mut callback_read_proof = HashOnly::::new(buffer); - let hash = if !proof { - // full node - let iter = children.filter_map(|v| v); // TODO assert same number as count - crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof) - } else { - // proof node, UsizeKeyNode should be big enough for hasher hybrid - // case. - let iter_key = seq_trie.iter_path_node_key::(None); - let iter = children - .zip(iter_key) - .filter_map(|(value, key)| if let Some(value) = value { - Some((key, value)) - } else { - None - }); - if let Some(hash) = crate::trie_root_from_proof( - &seq_trie, - iter, - additional_hashes, - &mut callback_read_proof, - false, - ) { - hash + // proof node, UsizeKeyNode should be big enough for hasher hybrid + // case. + let iter_key = seq_trie.iter_path_node_key::(None); + let iter = children + .zip(iter_key) + .filter_map(|(value, key)| if let Some(value) = value { + Some((key, value)) } else { - return None; - } + None + }); + let hash = if let Some(hash) = crate::trie_root_from_proof( + &seq_trie, + iter, + additional_hashes, + &mut callback_read_proof, + false, + ) { + hash + } else { + return None; }; let mut hash_buf2 = ::init_buffer(); ::buffer_hash(&mut hash_buf2, header); ::buffer_hash(&mut hash_buf2, hash.as_ref()); Some(H::buffer_finalize(&mut hash_buf2)) } + + fn hash_hybrid< + I: Iterator::Out>>, + >( + header: &[u8], + nb_children: usize, + children: I, + buffer: &mut HH::Buffer, + ) -> H::Out { + let seq_trie = SequenceBinaryTree::new(0, nb_children); + + let mut callback_read_proof = HashOnly::::new(buffer); + // full node + let iter = children.filter_map(|v| v); // TODO assert same number as count + let hash = crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof); + let mut hash_buf2 = ::init_buffer(); + ::buffer_hash(&mut hash_buf2, header); + ::buffer_hash(&mut hash_buf2, hash.as_ref()); + H::buffer_finalize(&mut hash_buf2) + } + } impl BinaryHasher for OrderedTrieHasher { diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index e7490edf..aae7fd8b 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -464,10 +464,8 @@ impl<'a, H: HasherHybrid, V, DB: HashDBHybrid> ProcessEncodedNode< ProcessEncodedNode<::Out> for TrieRootHybrid< common.header(&encoded_node[..]), nb_children, iter, - EmptyIter::default(), - false, &mut self.buffer, - ).expect("This node encoding is trusted") + ) } else { ::hash(&encoded_node[..]) }; @@ -691,10 +687,8 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashe common.header(&encoded_node[..]), nb_children, iter, - EmptyIter::default(), - false, &mut self.buffer, - ).expect("only proof fails: TODO two primitives") + ) } else { ::hash(&encoded_node[..]) }; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 458b472a..f540624a 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -20,7 +20,7 @@ use crate::node::{Node, NodePlan}; use crate::ChildReference; use hash_db::{HasherHybrid, BinaryHasher}; -use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, EmptyIter, ops::Range}; +use crate::rstd::{borrow::Borrow, Error, hash, vec::Vec, ops::Range}; /// Representation of a nible slice (right aligned). /// It contains a right aligned padded first byte (first pair element is the number of nibbles @@ -229,7 +229,7 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { children: &[Option>], common: ChildProofHeader, buffer: &mut ::Buffer, - ) -> Option; + ) -> H::Out; } impl> HashDBHybridDyn for C { @@ -240,7 +240,7 @@ impl> HashDBHybridDyn for C { children: &[Option>], common: ChildProofHeader, buffer: &mut ::Buffer, - ) -> Option { + ) -> H::Out { // TODO factor this with iter_build (just use the trait) also use in adapter from codec let nb_children = children.iter().filter(|v| v.is_some()).count(); @@ -257,8 +257,6 @@ impl> HashDBHybridDyn for C { common.header(value), nb_children, children, - EmptyIter::default(), - false, buffer, ) } @@ -390,10 +388,9 @@ impl Iterator for HashesPlan { } /// Adapter standard implementation to use with `HashDBInsertComplex` function. -/// TODO EMCH seems unused (there is test but??) pub fn hybrid_hash_node_adapter, Hasher: HasherHybrid>( encoded_node: &[u8] -) -> crate::rstd::result::Result, ()> { +) -> crate::rstd::result::Result, ()> { // TODO EMCH why returning result? Ok(if let Some((node, common)) = Codec::need_hybrid_proof(encoded_node) { match node { NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { @@ -409,10 +406,8 @@ pub fn hybrid_hash_node_adapter, H common.header(encoded_node), nb_children, children, - EmptyIter::default(), - false, // not a proof, will not fail &mut buf, - ).expect("not a proof, does not fail")) // TODO EMCH split function in two variants! + )) }, _ => unreachable!("hybrid only touch branch node"), } diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index 779637f0..8506206f 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -506,12 +506,11 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } }); - if let Some(h) = OrderedTrieHasher::::InnerHasher>::hash_hybrid( + if let Some(h) = OrderedTrieHasher::::InnerHasher>::hash_hybrid_proof( &common.header(node_data.as_slice())[..], nb_children, children, additional_hash.into_iter(), - true, hybrid_buf.as_mut().expect("Initialized for hybrid above"), ) { h diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 8082fcfc..5e35729f 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -575,14 +575,13 @@ pub fn decode_compact(db: &mut DB, encoded: &[Vec]) None } }); - if let Some(hash) = db.insert_branch_hybrid( + if let Some(hash) = db.insert_branch_hybrid_proof( prefix.as_prefix(), &node_data[..], common.header(&node_data[..]), nb_children, children, additional_hashes, - true, hybrid_buf.as_mut().expect("Initialized for hybrid above"), ) { hash diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 560963cd..e17ef059 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1488,7 +1488,7 @@ where &children[..], no_node, self.hybrid_hash_buffer.as_mut().expect("Lazy init above"), - ).expect("Trusted encoded value (was change earlier)") + ) } else { *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root[..]); } @@ -1548,7 +1548,7 @@ where &children[..], no_node, self.hybrid_hash_buffer.as_mut().expect("Lazy init above"), - ).expect("Trusted encoded value (was change earlier)") + ) } else { self.db.insert(prefix.as_prefix(), &encoded[..]) }; From f72d451019a957d1c64dc9f53bcffa85f83af8d3 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 16:23:22 +0200 Subject: [PATCH 41/73] removing bit of redundancy in api --- trie-db/src/proof/generate.rs | 28 +++++++++++++++++----------- trie-db/src/trie_codec.rs | 26 ++++++++++++-------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 280c8620..27be78ea 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -86,8 +86,7 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> /// Encode this entry to an encoded trie node with data properly omitted. fn encode_node( mut self, - hybrid: bool, - hash_buf: &mut ::Buffer, + hybrid: &mut Option<::Buffer>, ) -> TrieResult, C::HashOut, C::Error> { let node_data = self.node.data(); Ok(match self.node.node_plan() { @@ -113,6 +112,8 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> ) } NodePlan::Branch { value, children } => { + let mut hash_buf = hybrid.as_mut(); + let hybrid = hash_buf.is_some(); Self::complete_branch_children( node_data, children, @@ -127,7 +128,7 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> C::encode_compact_proof::( hash_proof_header, &self.children[..], - hash_buf, + hash_buf.as_mut().expect("hybrid is true"), &self.in_proof_children[..], ) } else { @@ -138,6 +139,8 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> } }, NodePlan::NibbledBranch { partial: partial_plan, value, children } => { + let mut hash_buf = hybrid.as_mut(); + let hybrid = hash_buf.is_some(); let partial = partial_plan.build(node_data); Self::complete_branch_children( node_data, @@ -155,7 +158,7 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> C::encode_compact_proof::( hash_proof_header, &self.children[..], - hash_buf, + hash_buf.as_mut().expect("hybrid is true"), &self.in_proof_children[..], ) } else { @@ -277,8 +280,12 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) .collect::>(); keys.sort(); keys.dedup(); - let mut hash_buf = <::InnerHasher as hash_db::BinaryHasher>::init_buffer(); - let hash_buf = &mut hash_buf; + let mut hybrid = if L::HYBRID_HASH { + Some(<::InnerHasher as hash_db::BinaryHasher>::init_buffer()) + } else { + None + }; + let hybrid = &mut hybrid; // The stack of nodes through a path in the trie. Each entry is a child node of the preceding @@ -292,7 +299,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) let key = LeftNibbleSlice::new(key_bytes); // Unwind the stack until the new entry is a child of the last entry on the stack. - unwind_stack(&mut stack, &mut proof_nodes, Some(&key), L::HYBRID_HASH, hash_buf)?; + unwind_stack(&mut stack, &mut proof_nodes, Some(&key), hybrid)?; // Perform the trie lookup for the next key, recording the sequence of nodes traversed. let mut recorder = Recorder::new(); @@ -400,7 +407,7 @@ pub fn generate_proof<'a, T, L, I, K>(trie: &T, keys: I) } } - unwind_stack(&mut stack, &mut proof_nodes, None, L::HYBRID_HASH, hash_buf)?; + unwind_stack(&mut stack, &mut proof_nodes, None, hybrid)?; Ok(proof_nodes) } @@ -550,8 +557,7 @@ fn unwind_stack( stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, - hybrid: bool, - hash_buf: &mut ::Buffer, + hybrid: &mut Option<::Buffer>, ) -> TrieResult<(), C::HashOut, C::Error> where H: HasherHybrid, @@ -566,7 +572,7 @@ fn unwind_stack( _ => { // Pop and finalize node from the stack. let index = entry.output_index; - let encoded = entry.encode_node(hybrid, hash_buf)?; + let encoded = entry.encode_node(hybrid)?; if let Some(parent_entry) = stack.last_mut() { parent_entry.set_child(&encoded); } diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 5e35729f..72d27f83 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -103,8 +103,7 @@ impl EncoderStackEntry { /// Generates the encoding of the subtrie rooted at this entry. fn encode_node( &self, - hybrid_hash: bool, - hash_buf: &mut H::Buffer, + hybrid: &mut Option, ) -> Result, C::HashOut, C::Error> where H: HasherHybrid, @@ -123,7 +122,7 @@ impl EncoderStackEntry { }, NodePlan::Branch { value, children } => { let children = Self::branch_children(node_data, &children, &self.omit_children[..])?; - if hybrid_hash { + if let Some(hash_buf) = hybrid.as_mut() { let hash_proof_header = C::branch_node_for_hash( children.iter(), value.clone().map(|range| &node_data[range]), @@ -144,7 +143,7 @@ impl EncoderStackEntry { NodePlan::NibbledBranch { partial, value, children } => { let children = Self::branch_children(node_data, &children, &self.omit_children[..])?; let partial = partial.build(node_data); - if hybrid_hash { + if let Some(hash_buf) = hybrid.as_mut() { let hash_proof_header = C::branch_node_nibbled_for_hash( partial.right_iter(), partial.len(), @@ -214,9 +213,12 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE { let mut output = Vec::new(); - // TODO make it optional and replace boolean is_hybrid - let mut hash_buf = ::init_buffer(); - let hash_buf = &mut hash_buf; + let mut hybrid = if L::HYBRID_HASH { + Some(::init_buffer()) + } else { + None + }; + let hybrid = &mut hybrid; // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. @@ -250,8 +252,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE // unstack as in not inline case if let Some(last_entry) = stack.pop() { output[last_entry.output_index] = last_entry.encode_node::( - L::HYBRID_HASH, - hash_buf, + hybrid, )?; } } @@ -281,8 +282,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE break; } else { output[last_entry.output_index] = last_entry.encode_node::( - L::HYBRID_HASH, - hash_buf, + hybrid, )?; } } @@ -312,8 +312,7 @@ pub fn encode_compact(db: &TrieDB) -> Result>, TrieHash, CE while let Some(entry) = stack.pop() { output[entry.output_index] = entry.encode_node::( - L::HYBRID_HASH, - hash_buf, + hybrid, )?; } @@ -634,7 +633,6 @@ pub fn binary_additional_hashes( None }); - let mut callback = HashProof::::new(hash_buf, to_prove, true); let hashes = children.iter() .filter_map(|o_child| o_child.as_ref()) From 9ce822678aa92b8ff5d5339dc6475087fb225851 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 16:36:23 +0200 Subject: [PATCH 42/73] removed unused offset field --- ordered-trie/src/lib.rs | 38 ++++++++++++++++---------------------- trie-db/src/trie_codec.rs | 2 +- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index da30d6f3..82f514ac 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -54,8 +54,6 @@ pub type DBValue = Vec; pub struct SequenceBinaryTree { // Metadata (needs to be validated) - /// global offset for index. - offset: K, /// Nb deleted values at end. end: K, end_depth: usize, @@ -71,7 +69,6 @@ pub struct SequenceBinaryTree { impl Default for SequenceBinaryTree { fn default() -> Self { SequenceBinaryTree { - offset: 0, end: 0, end_depth: 0, depth: 0, @@ -95,11 +92,10 @@ fn right_at(value: usize, index: usize) -> bool { } impl SequenceBinaryTree { - pub fn new(offset: usize, number: usize) -> Self { + pub fn new(number: usize) -> Self { let len = number; if len == 0 { SequenceBinaryTree { - offset, end: 0, end_depth: 0, depth: 0, @@ -112,7 +108,6 @@ impl SequenceBinaryTree { let end_depth = depth(end); let depth = depth(length - 1); SequenceBinaryTree { - offset, end, end_depth, depth, @@ -122,7 +117,6 @@ impl SequenceBinaryTree { } } - // TODO consider storing that fn nb_elements(&self) -> usize { self.length - self.end } @@ -133,7 +127,7 @@ impl SequenceBinaryTree { return; } if self.length == 0 { - *self = Self::new(self.offset, nb); + *self = Self::new(nb); return; } while nb > self.end { @@ -922,7 +916,7 @@ impl> HasherHybrid for OrderedTr additional_hashes: I2, buffer: &mut HH::Buffer, ) -> Option { - let seq_trie = SequenceBinaryTree::new(0, nb_children); + let seq_trie = SequenceBinaryTree::new(nb_children); let mut callback_read_proof = HashOnly::::new(buffer); // proof node, UsizeKeyNode should be big enough for hasher hybrid @@ -960,7 +954,7 @@ impl> HasherHybrid for OrderedTr children: I, buffer: &mut HH::Buffer, ) -> H::Out { - let seq_trie = SequenceBinaryTree::new(0, nb_children); + let seq_trie = SequenceBinaryTree::new(nb_children); let mut callback_read_proof = HashOnly::::new(buffer); // full node @@ -1095,7 +1089,7 @@ mod test { prev = nb; tree.push(inc); assert_eq!(tree.depth, depth); - let tree2 = Tree::new(0, nb); + let tree2 = Tree::new(nb); assert_eq!(tree2.depth, depth); assert_eq!(tree, tree2); } @@ -1104,33 +1098,33 @@ mod test { #[test] fn test_depth_index() { // 8 trie - let tree = Tree::new(0, 7); + let tree = Tree::new(7); assert_eq!(tree.depth_index(3), 3); assert_eq!(tree.depth_index(4), 3); assert_eq!(tree.depth_index(6), 2); - let tree = Tree::new(0, 6); + let tree = Tree::new(6); assert_eq!(tree.depth_index(0), 3); assert_eq!(tree.depth_index(3), 3); assert_eq!(tree.depth_index(4), 2); assert_eq!(tree.depth_index(5), 2); - let tree = Tree::new(0, 5); + let tree = Tree::new(5); assert_eq!(tree.depth_index(3), 3); assert_eq!(tree.depth_index(4), 1); // 16 trie - let tree = Tree::new(0, 12); + let tree = Tree::new(12); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 3); assert_eq!(tree.depth_index(11), 3); - let tree = Tree::new(0, 11); + let tree = Tree::new(11); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 3); assert_eq!(tree.depth_index(9), 3); assert_eq!(tree.depth_index(10), 2); - let tree = Tree::new(0, 10); + let tree = Tree::new(10); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 2); assert_eq!(tree.depth_index(9), 2); - let tree = Tree::new(0, 9); + let tree = Tree::new(9); assert_eq!(tree.depth_index(7), 4); assert_eq!(tree.depth_index(8), 1); // 32 trie TODO @@ -1142,7 +1136,7 @@ mod test { // let cases = [7, 6, 5, 12, 11, 10, 9]; for nb in 0usize..16 { let mut n = 0; - let tree = Tree::new(0, nb); + let tree = Tree::new(nb); for (i, (d, k)) in tree.iter_depth(None) .zip(tree.iter_path_node_key::(None)) .enumerate() { @@ -1281,7 +1275,7 @@ mod test { fn test_hash_only() { let result = base16_roots(); for l in 0..17 { - let tree = Tree::new(0, l); + let tree = Tree::new(l); let mut hash_buf = ::init_buffer(); let mut callback = HashOnly::::new(&mut hash_buf); let hashes: Vec<_> = hashes(l); @@ -1294,7 +1288,7 @@ mod test { fn test_one_element_proof() { let result = base16_roots(); for l in 0..17 { - let tree = Tree::new(0, l); + let tree = Tree::new(l); let mut hash_buf = ::init_buffer(); let mut hash_buf2 = ::init_buffer(); let mut hash_buf3 = ::init_buffer(); @@ -1348,7 +1342,7 @@ mod test { for (l, ps) in tests.iter() { let l = *l; let ps = *ps; - let tree = Tree::new(0, l); + let tree = Tree::new(l); let mut hash_buf = ::init_buffer(); let mut hash_buf2 = ::init_buffer(); let mut hash_buf3 = ::init_buffer(); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 72d27f83..fd947c27 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -621,7 +621,7 @@ pub fn binary_additional_hashes( hash_buf: &mut H::Buffer, ) -> Vec { let nb_children = children.iter().filter(|v| v.is_some()).count(); - let tree = SequenceBinaryTree::new(0, nb_children); + let tree = SequenceBinaryTree::new(nb_children); let to_prove = children.iter().zip(in_proof_children.iter()) .filter_map(|(o_child, in_proof)| o_child.as_ref().map(|_| *in_proof)) From 256892209397215d7345e9b5534235c653e2cf5c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 16:37:12 +0200 Subject: [PATCH 43/73] removed unused offset field --- ordered-trie/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 82f514ac..3420645a 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -121,8 +121,8 @@ impl SequenceBinaryTree { self.length - self.end } - #[cfg(test)] // TODO is in implementation but unused, should be rename to resize - fn push(&mut self, mut nb: usize) { + #[cfg(test)] + fn resize(&mut self, mut nb: usize) { if nb == 0 { return; } From 694af92e988c59e0674044dfd622faa43abcad27 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 16:54:34 +0200 Subject: [PATCH 44/73] ordered trie more cleaning --- ordered-trie/src/lib.rs | 30 +++++++++++------------------- trie-db/src/trie_codec.rs | 2 +- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 3420645a..63b96c3b 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -16,7 +16,7 @@ //! This crate contains implementation of trie/tree based on ordered sequential key only. //! -//! Targetted use case is a stack or a fifo. +//! Current use case is a fixed length tree. #[cfg(not(feature = "std"))] extern crate alloc; @@ -80,11 +80,6 @@ impl Default for SequenceBinaryTree { fn depth(nb: usize) -> usize { (0usize.leading_zeros() - nb.leading_zeros()) as usize -/* if nb == 0 { - return 0; - } - - ((0usize.leading_zeros() - (nb - 1).leading_zeros()) as usize) + 1*/ } fn right_at(value: usize, index: usize) -> bool { @@ -122,7 +117,7 @@ impl SequenceBinaryTree { } #[cfg(test)] - fn resize(&mut self, mut nb: usize) { + fn resize_append(&mut self, mut nb: usize) { if nb == 0 { return; } @@ -140,7 +135,6 @@ impl SequenceBinaryTree { self.end = self.length; self.length *= 2; } - } self.end -= nb; self.end_depth = depth(self.end); @@ -151,7 +145,8 @@ impl SequenceBinaryTree { if !tmp == 0 { let mut nb_skip = 0; for i in 0..self.end_depth { - let ix = self.end_depth - i - 1; // - 1 from the fact that main depth is 1 less due to redundancy of first level (depth in number of change of level) + // -1 related to redundancy of first level of main depth (depth in number of change of level) + let ix = self.end_depth - i - 1; if self.end & (1 << ix) != 0 { // this is a skip nb_skip += 1; @@ -174,7 +169,8 @@ impl SequenceBinaryTree { if !tmp == 0 { let mut result: KN = (index, self.depth).into(); for i in 0..self.end_depth { - let ix = self.end_depth - i - 1; // - 1 from the fact that main depth is 1 less due to redundancy of first level (depth in number of change of level) + // -1 related to redundancy of first level of main depth (depth in number of change of level) + let ix = self.end_depth - i - 1; if self.end & (1 << ix) != 0 { // this is a skip let ix = result.depth() - ix - 1; @@ -227,15 +223,11 @@ impl SequenceBinaryTree { }) } - pub fn iter_path_node_key(&self, from: Option) -> impl Iterator + pub fn iter_path_node_key(&self) -> impl Iterator where KN: KeyNode + From<(usize, usize)> + Clone, { - if let Some(_from) = from { - unimplemented!(); - } let nb_elements = self.nb_elements(); - // TODO index should not be use but key.value, this is double counting things let mut index = 0; let length = self.length; let mut end = KN::from((self.end, self.end_depth)); @@ -758,7 +750,7 @@ pub fn trie_root(layout: &SequenceBinaryTree, input: I, cal I: Iterator, F: ProcessNode, { - let mut iter = input.into_iter().zip(layout.iter_path_node_key::(None)); + let mut iter = input.into_iter().zip(layout.iter_path_node_key::()); debug_assert!({ let (r, s) = iter.size_hint(); if s == Some(r) { @@ -921,7 +913,7 @@ impl> HasherHybrid for OrderedTr let mut callback_read_proof = HashOnly::::new(buffer); // proof node, UsizeKeyNode should be big enough for hasher hybrid // case. - let iter_key = seq_trie.iter_path_node_key::(None); + let iter_key = seq_trie.iter_path_node_key::(); let iter = children .zip(iter_key) .filter_map(|(value, key)| if let Some(value) = value { @@ -1087,7 +1079,7 @@ mod test { for (nb, depth) in values.iter().cloned() { let inc = nb - prev; prev = nb; - tree.push(inc); + tree.resize_append(inc); assert_eq!(tree.depth, depth); let tree2 = Tree::new(nb); assert_eq!(tree2.depth, depth); @@ -1138,7 +1130,7 @@ mod test { let mut n = 0; let tree = Tree::new(nb); for (i, (d, k)) in tree.iter_depth(None) - .zip(tree.iter_path_node_key::(None)) + .zip(tree.iter_path_node_key::()) .enumerate() { n += 1; assert_eq!(d, tree.depth_index(i)); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index fd947c27..9152e1af 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -626,7 +626,7 @@ pub fn binary_additional_hashes( let to_prove = children.iter().zip(in_proof_children.iter()) .filter_map(|(o_child, in_proof)| o_child.as_ref().map(|_| *in_proof)) // correct iteration over binary tree - .zip(tree.iter_path_node_key::(None)) + .zip(tree.iter_path_node_key::()) .filter_map(|(in_proof, ix_key)| if in_proof { Some(ix_key) } else { From 1fbe9e65befbe829cc4b2c6b0f8d57e5590486ec Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 17:08:28 +0200 Subject: [PATCH 45/73] ordered trie more cleaning --- ordered-trie/src/lib.rs | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/ordered-trie/src/lib.rs b/ordered-trie/src/lib.rs index 63b96c3b..bc4e136b 100644 --- a/ordered-trie/src/lib.rs +++ b/ordered-trie/src/lib.rs @@ -396,8 +396,6 @@ impl KeyNode for UsizeKeyNode { if self.depth == 0 { return None; } - // TODO is pop returned value of any use: - // most likely not -> change trait and test let result = self.value & 1; self.depth -= 1; self.value = self.value >> 1; @@ -412,8 +410,6 @@ impl KeyNode for UsizeKeyNode { if self.depth == 0 { return None; } - // TODO is pop returned value of any use: - // most likely not -> change trait and test let result = self.value & (1 << (self.depth - 1)); self.value = self.value & !(1 << (self.depth - 1)); self.depth -= 1; @@ -604,11 +600,14 @@ impl MultiProofState { } } // from stack - if let Some((stack_hash, _stack_join)) = self.stack.pop() { + if let Some((stack_hash, stack_join)) = self.stack.pop() { // current is dropped. self.current_key = Some(stack_hash); - // TODO check if stack depth == this new depth (should be?). - self.refresh_join1(); + debug_assert!({ + self.refresh_join1(); + Some(stack_join) == self.join1 + }); + self.join1 = Some(stack_join); } else { // fuse last interval self.join1 = None; @@ -653,7 +652,12 @@ impl MultiProofState { self.current_key = self.next_key1.take(); self.next_key1 = self.next_key2.take(); self.next_key2 = next_keys.next(); - self.refresh_join1(); // TODO could also use join2 for join1 would be fastest + + debug_assert!({ + self.refresh_join1(); + self.join1 == self.join2 + }); + self.join1 = self.join2; self.refresh_join2(); } else { // keep interval @@ -678,9 +682,6 @@ impl MultiProofState { } impl<'a, H: BinaryHasher, KN: KeyNode, I: Iterator> HashProof<'a, H, I, KN> { - // TODO write function to build from iter of unchecked usize indexes: map iter - // with either depth_at or the depth_iterator skipping undesired elements (second - // seems better as it filters out of range. pub fn new( buffer: &'a mut H::Buffer, mut to_prove: I, @@ -838,8 +839,7 @@ pub fn trie_root_from_proof( if let Some(c) = items.next() { current = c; } else { - // TODO check if we even can produce such proof. - // no item case root is directly in additional + // when no item, root is first from additional hashes if let Some(h) = additional_hash.next() { if allow_additionals_hashes || additional_hash.next().is_none() { callback.register_root(&h); @@ -866,7 +866,7 @@ pub fn trie_root_from_proof( } if Some(depth) == common_depth { let (key_next, hash_next) = next.take().expect("common depth is some"); - stack.push((current, depth)); // TODO process sibling without stack? or all on stack + stack.push((current, depth)); current = (key_next, hash_next); next = items.next(); common_depth = calc_common_depth(¤t, &next); @@ -949,8 +949,7 @@ impl> HasherHybrid for OrderedTr let seq_trie = SequenceBinaryTree::new(nb_children); let mut callback_read_proof = HashOnly::::new(buffer); - // full node - let iter = children.filter_map(|v| v); // TODO assert same number as count + let iter = children.filter_map(|v| v); let hash = crate::trie_root::<_, UsizeKeyNode, _, _>(&seq_trie, iter, &mut callback_read_proof); let mut hash_buf2 = ::init_buffer(); ::buffer_hash(&mut hash_buf2, header); @@ -1125,7 +1124,6 @@ mod test { #[test] fn test_depth_iter() { // cases corresponding to test_depth, TODO add more -// let cases = [7, 6, 5, 12, 11, 10, 9]; for nb in 0usize..16 { let mut n = 0; let tree = Tree::new(nb); From d94d2e15badbfc09d8a25de89052c35a9b98c57a Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 17:52:31 +0200 Subject: [PATCH 46/73] removing access to node range --- test-support/keccak-hasher/src/lib.rs | 23 ----------------------- test-support/reference-trie/src/lib.rs | 2 -- trie-db/src/iter_build.rs | 4 ---- trie-db/src/node.rs | 14 +++++++++----- trie-db/src/node_codec.rs | 5 +---- 5 files changed, 10 insertions(+), 38 deletions(-) diff --git a/test-support/keccak-hasher/src/lib.rs b/test-support/keccak-hasher/src/lib.rs index cb7a4aec..8f988d73 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/test-support/keccak-hasher/src/lib.rs @@ -67,29 +67,6 @@ fn test_keccack_hasher() { hash_db::test_binary_hasher::() } -/* TODO this is rather bad trait see if delete?? -#[derive(Default, Debug, Clone, PartialEq)] -pub struct FixKeccakHasher([u8;32]); -impl FixHash for FixKeccakHasher { - type Hasher = KeccakHasher; - const NEED_FIRST_HASHED: bool = true; - const EMPTY_HASHES: &'static [&'static [u8]] = &[]; - - fn new(first: ::Out) -> Self { - FixKeccakHasher(first) - } - fn hash(&mut self, second: &::Out) { - unimplemented!() - } - fn current_state(&self) -> &::Out { - &self.0 - } - fn finalize(self) -> ::Out { - unimplemented!() - } -} -*/ - #[cfg(test)] mod tests { use super::*; diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 23228ed3..05a81887 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -1020,8 +1020,6 @@ fn encode_proof_internal( if let Some(ChildReference::Inline(h, nb)) = child.borrow() { if *nb > 0 { if in_proof[ix] { - // TODO do not write inline of null size, these are defined - // in the bitmap and from the algos. debug_assert!(*nb < 128); result.push(*nb as u8); result.push(ix as u8); diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index aae7fd8b..63fb8639 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -416,7 +416,6 @@ impl<'a, H: Hasher, V, DB: HashDB> ProcessEncodedNode<::Out> prefix: Prefix, (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, - // TODO different trait?? _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); @@ -495,7 +494,6 @@ impl ProcessEncodedNode<::Out> for TrieRoot, ChildProofHeader), is_root: bool, - // TODO different trait _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); @@ -615,7 +613,6 @@ impl ProcessEncodedNode<::Out> for TrieRootPrint, ChildProofHeader), is_root: bool, - // TODO different trait? _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { println!("Encoded node: {:x?}", &encoded_node); @@ -643,7 +640,6 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashed { _: Prefix, (encoded_node, _common): (Vec, ChildProofHeader), is_root: bool, - // TODO different trait _hybrid_hash: Option<(impl Iterator>>>, usize)>, ) -> ChildReference<::Out> { let len = encoded_node.len(); diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 66f95e01..6dda4101 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -85,14 +85,18 @@ impl NodeHandlePlan { NodeHandlePlan::Inline(range) => NodeHandle::Inline(&data[range.clone()]), } } + /// Range of node handle definition in encoded data. - /// TODO EMCH this breaks design a bit, maybe return H::Out - /// or put a util function. - pub fn range<'a, 'b>(&'a self) -> Range { + pub fn as_hash<'a, 'b, HOut: Default + AsMut<[u8]>>(&'a self, encoded_node: &[u8]) -> HOut { + let mut dest = HOut::default(); + match self { - NodeHandlePlan::Hash(range) => range.clone(), - NodeHandlePlan::Inline(range) => range.clone(), + NodeHandlePlan::Inline(range) + | NodeHandlePlan::Hash(range) => { + dest.as_mut()[..range.len()].copy_from_slice(&encoded_node[range.clone()]); + }, } + dest } /// Check if it is an inline node. diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index f540624a..932555bf 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -396,10 +396,7 @@ pub fn hybrid_hash_node_adapter, H NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { let nb_children = children.iter().filter(|v| v.is_some()).count(); let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { - let range = range.range(); - let mut dest = Hasher::Out::default(); - dest.as_mut()[..range.len()].copy_from_slice(&encoded_node[range]); - dest + range.as_hash(encoded_node) })); let mut buf = ::InnerHasher::init_buffer(); Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( From e965a2dfe75ed7aec642f4349c5e576f304c8887 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 18:26:31 +0200 Subject: [PATCH 47/73] revert trait changes --- test-support/reference-trie/src/lib.rs | 38 ++++++++-------- trie-db/src/node_codec.rs | 60 ++++++++++++-------------- trie-db/src/proof/generate.rs | 4 +- trie-db/src/trie_codec.rs | 4 +- 4 files changed, 49 insertions(+), 57 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 05a81887..9611b947 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -858,25 +858,24 @@ impl NodeCodecHybrid for ReferenceNodeCodec { fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], - hash_buf: &mut BH::Buffer, in_proof: &[bool], + hash_buf: &mut BH::Buffer, ) -> Vec { encode_proof_internal::(hash_proof_header, children, hash_buf, in_proof) } - fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)> { + fn need_hybrid_proof(data: &[u8]) -> Result, ()> { if data.len() > 0 { if NodeHeader::is_branch(data[0]) { - if let Ok((node, offset)) = Self::decode_plan_internal(data, false) { - let header = ChildProofHeader::Range( Range { - start: 0, - end: offset, - }); - return Some((node, header)) - } + let (node, offset) = Self::decode_plan_internal(data, false).map_err(|_| ())?; + let header = ChildProofHeader::Range( Range { + start: 0, + end: offset, + }); + return Ok(Some((node, header))) } } - None + Ok(None) } fn codec_error(desc: &'static str) -> Self::Error { @@ -1320,25 +1319,24 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], - hash_buf: &mut BH::Buffer, in_proof: &[bool], + hash_buf: &mut BH::Buffer, ) -> Vec { encode_proof_internal::(hash_proof_header, children, hash_buf, in_proof) } - fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)> { + fn need_hybrid_proof(data: &[u8]) -> Result, ()> { if data.len() > 0 { if NodeHeaderNoExt::is_branch(data[0]) { - if let Ok((node, offset)) = Self::decode_plan_internal(data, false) { - let header = ChildProofHeader::Range( Range { - start: 0, - end: offset, - }); - return Some((node, header)) - } + let (node, offset) = Self::decode_plan_internal(data, false).map_err(|_| ())?; + let header = ChildProofHeader::Range( Range { + start: 0, + end: offset, + }); + return Ok(Some((node, header))) } } - None + Ok(None) } fn codec_error(desc: &'static str) -> Self::Error { diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 932555bf..6db88090 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -45,8 +45,6 @@ pub trait NodeCodec: Sized { /// Decode bytes to a `Node`. Returns `Self::E` on failure. fn decode(data: &[u8]) -> Result { - // TODO ensure real use codec have their own implementation - // as this can be slower Ok(Self::decode_plan(data)?.build(data)) } @@ -106,7 +104,7 @@ pub trait NodeCodecHybrid: NodeCodec { /// are not use by the proof, in this case they are stored as an inline /// zero length child. /// - /// With resulting node is attached a bitmap of children included in + /// With resulting node is attached a bitmap of children included in /// the proof calculation and a sequence of additianal node for proof /// verification. /// In practice this contains inline node that are in the proof and @@ -137,19 +135,18 @@ pub trait NodeCodecHybrid: NodeCodec { fn encode_compact_proof( hash_proof_header: Vec, children: &[Option>], - // TODO EMCH siwtch position on 2 last params - hash_buf: &mut H::Buffer, in_proof: &[bool], + hash_buf: &mut H::Buffer, ) -> Vec; /// Does the encoded content need a hybrid proof. /// With current hybrid proof capability this is strictly /// the same as checking if the node is a branch. /// It return the proof header and the NodePlan if hybrid proof is needed. - fn need_hybrid_proof(data: &[u8]) -> Option<(NodePlan, ChildProofHeader)>; + fn need_hybrid_proof(data: &[u8]) -> crate::rstd::result::Result, ()>; /// Returns branch node encoded for storage, and additional information for hash calculation. - /// + /// /// Takes an iterator yielding `ChildReference` and an optional value /// as input, the third input is an output container needed for hash calculation. fn branch_node_common( @@ -219,9 +216,6 @@ pub trait HashDBHybridDyn: Send + Sync + HashDB { /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. - /// - /// TODO warn semantic of children differs from HashDBHybrid (in HashDBHybrid it is the - /// children of the binary hasher, here it is the children of the patricia merkle trie). fn insert_branch_hybrid( &mut self, prefix: Prefix, @@ -241,8 +235,6 @@ impl> HashDBHybridDyn for C { common: ChildProofHeader, buffer: &mut ::Buffer, ) -> H::Out { - - // TODO factor this with iter_build (just use the trait) also use in adapter from codec let nb_children = children.iter().filter(|v| v.is_some()).count(); let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { let mut dest = H::Out::default(); @@ -352,7 +344,7 @@ impl<'a, I, HO: Default> HashesIter<'a, I, HO> { impl<'a, I, HO> Iterator for HashesIter<'a, I, HO> where I: Iterator>, - HO: AsMut<[u8]> + Clone, + HO: AsMut<[u8]> + Clone, { type Item = HO; @@ -390,27 +382,29 @@ impl Iterator for HashesPlan { /// Adapter standard implementation to use with `HashDBInsertComplex` function. pub fn hybrid_hash_node_adapter, Hasher: HasherHybrid>( encoded_node: &[u8] -) -> crate::rstd::result::Result, ()> { // TODO EMCH why returning result? - Ok(if let Some((node, common)) = Codec::need_hybrid_proof(encoded_node) { - match node { - NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { - let nb_children = children.iter().filter(|v| v.is_some()).count(); - let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { - range.as_hash(encoded_node) - })); - let mut buf = ::InnerHasher::init_buffer(); - Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( - common.header(encoded_node), - nb_children, - children, - &mut buf, - )) - }, - _ => unreachable!("hybrid only touch branch node"), +) -> crate::rstd::result::Result, ()> { + Codec::need_hybrid_proof(encoded_node).map(|hybrid| + if let Some((node, common)) = hybrid { + match node { + NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { + range.as_hash(encoded_node) + })); + let mut buf = ::InnerHasher::init_buffer(); + Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( + common.header(encoded_node), + nb_children, + children, + &mut buf, + )) + }, + _ => unreachable!("hybrid only touch branch node"), + } + } else { + None } - } else { - None - }) + ) } #[test] diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 27be78ea..f2c1c62d 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -128,8 +128,8 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> C::encode_compact_proof::( hash_proof_header, &self.children[..], - hash_buf.as_mut().expect("hybrid is true"), &self.in_proof_children[..], + hash_buf.as_mut().expect("hybrid is true"), ) } else { C::branch_node( @@ -158,8 +158,8 @@ impl<'a, C: NodeCodecHybrid, H: HasherHybrid> StackEntry<'a, C, H> C::encode_compact_proof::( hash_proof_header, &self.children[..], - hash_buf.as_mut().expect("hybrid is true"), &self.in_proof_children[..], + hash_buf.as_mut().expect("hybrid is true"), ) } else { C::branch_node_nibbled( diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 9152e1af..3031b1d9 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -130,8 +130,8 @@ impl EncoderStackEntry { C::encode_compact_proof::( hash_proof_header, &children[..], - hash_buf, &self.in_proof_children[..], + hash_buf, ) } else { C::branch_node( @@ -153,8 +153,8 @@ impl EncoderStackEntry { C::encode_compact_proof::( hash_proof_header, &children[..], - hash_buf, &self.in_proof_children, + hash_buf, ) } else { C::branch_node_nibbled( From d91a03ba238659cf4c646b4698e7dd645b76c710 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 18:51:26 +0200 Subject: [PATCH 48/73] removing unused registration --- test-support/reference-trie/src/lib.rs | 26 +++++++++++++++----------- trie-db/src/iter_build.rs | 4 ++-- trie-db/src/node_codec.rs | 4 ++-- trie-db/src/proof/verify.rs | 17 ++++++----------- trie-db/src/trie_codec.rs | 4 ++-- trie-db/src/triedbmut.rs | 4 ++-- 6 files changed, 29 insertions(+), 30 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 9611b947..b3edb769 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -799,7 +799,7 @@ impl NodeCodec for ReferenceNodeCodec { children: impl Iterator>>>, maybe_value: Option<&[u8]>, ) -> Vec { - Self::branch_node_internal(children, maybe_value, None, true).0 + Self::branch_node_internal(children, maybe_value, None, false, true).0 } fn branch_node_nibbled( @@ -824,9 +824,9 @@ impl NodeCodecHybrid for ReferenceNodeCodec { fn branch_node_common( children: impl Iterator>>>, maybe_value: Option<&[u8]>, - register_children: &mut [Option>], + register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader) { - Self::branch_node_internal(children, maybe_value, Some(register_children), true) + Self::branch_node_internal(children, maybe_value, register_children, true, true) } fn branch_node_nibbled_common( @@ -834,7 +834,7 @@ impl NodeCodecHybrid for ReferenceNodeCodec { _number_nibble: usize, _children: impl Iterator>>>, _maybe_value: Option<&[u8]>, - _register_children: &mut [Option>], + _register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader) { unreachable!() } @@ -843,7 +843,7 @@ impl NodeCodecHybrid for ReferenceNodeCodec { children: impl Iterator>>>, maybe_value: Option<&[u8]>, ) -> Vec { - Self::branch_node_internal(children, maybe_value, None, false).0 + Self::branch_node_internal(children, maybe_value, None, true, false).0 } fn branch_node_nibbled_for_hash( @@ -888,6 +888,7 @@ impl ReferenceNodeCodec { children: impl Iterator::HashOut>>>>, maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, + hybrid: bool, encode_children: bool, ) -> (Vec, ChildProofHeader) { let mut output = vec![0; BITMAP_LENGTH + 1]; @@ -902,7 +903,7 @@ impl ReferenceNodeCodec { let ix = &mut ix; let mut register_children = register_children.as_mut(); let register_children = &mut register_children; - let common = if encode_children && register_children.is_some() { + let common = if encode_children && hybrid { ChildProofHeader::Range(Range { start: 0, end: output.len(), @@ -1168,7 +1169,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { children: impl Iterator>>>, maybe_value: Option<&[u8]>, ) -> Vec { - Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, None, true).0 + Self::branch_node_nibbled_internal(partial, number_nibble, children, maybe_value, None, false, true).0 } } @@ -1179,6 +1180,7 @@ impl ReferenceNodeCodecNoExt { children: impl Iterator::HashOut>>>>, maybe_value: Option<&[u8]>, mut register_children: Option<&mut [Option>]>, + hybrid: bool, encode_children: bool, ) -> (Vec, ChildProofHeader) { let mut output = if maybe_value.is_some() { @@ -1204,7 +1206,7 @@ impl ReferenceNodeCodecNoExt { let ix = &mut ix; let mut register_children = register_children.as_mut(); let register_children = &mut register_children; - let common = if encode_children && register_children.is_some() { + let common = if encode_children && hybrid { ChildProofHeader::Range(Range { start: 0, end: output.len(), @@ -1271,7 +1273,7 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { fn branch_node_common( _children: impl Iterator::Out>>>>, _maybe_value: Option<&[u8]>, - _register_children: &mut [Option>], + _register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader) { unreachable!() } @@ -1281,14 +1283,15 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { number_nibble: usize, children: impl Iterator>>>, maybe_value: Option<&[u8]>, - register_children: &mut [Option>], + register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader) { Self::branch_node_nibbled_internal( partial, number_nibble, children, maybe_value, - Some(register_children), + register_children, + true, true, ) } @@ -1312,6 +1315,7 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { children, maybe_value, None, + true, false, ).0 } diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 63fb8639..ada41f64 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -212,7 +212,7 @@ impl CacheAccum T::Codec::branch_node_common( self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - register_children.as_mut() + Some(register_children.as_mut()) ) } else { (T::Codec::branch_node( @@ -264,7 +264,7 @@ impl CacheAccum pr.right_range_iter(nkeyix.1), nkeyix.1, self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), - register_children.as_mut(), + Some(register_children.as_mut()), ) } else { (T::Codec::branch_node_nibbled( diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 6db88090..e354afb5 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -152,7 +152,7 @@ pub trait NodeCodecHybrid: NodeCodec { fn branch_node_common( children: impl Iterator>>>, value: Option<&[u8]>, - register_children: &mut [Option>], + register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader); /// Variant of `branch_node_common` but with a nibble. @@ -164,7 +164,7 @@ pub trait NodeCodecHybrid: NodeCodec { number_nibble: usize, children: impl Iterator>>>, value: Option<&[u8]>, - register_children: &mut [Option>], + register_children: Option<&mut [Option>]>, ) -> (Vec, ChildProofHeader); /// Returns branch node encoded information for hash. diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index 8506206f..a08d6f88 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -13,7 +13,7 @@ //! Verification of compact proofs for Merkle-Patricia tries. use crate::rstd::{ - convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec, vec::Vec, + convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec::Vec, }; use crate::{ CError, ChildReference, nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, @@ -107,7 +107,7 @@ struct StackEntry<'a, C: NodeCodecHybrid, H> { /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The child references to use in reconstructing the trie nodes. - children: Vec>>, + children: [Option>; NIBBLE_LENGTH], /// Proof info if a hybrid proof is needed. hybrid: Option<(Bitmap, HashesIter<'a, C::AdditionalHashesPlan, C::HashOut>)>, _marker: PhantomData<(C, H)>, @@ -120,9 +120,8 @@ impl<'a, C: NodeCodecHybrid, H: BinaryHasher> StackEntry<'a, C, H> fn new(node_data: &'a [u8], prefix: LeftNibbleSlice<'a>, is_inline: bool, hybrid: bool) -> Result> { - let children = vec![None; NIBBLE_LENGTH]; // TODO use array + let children = [None; NIBBLE_LENGTH]; let (node, hybrid) = if !is_inline && hybrid { - // TODO factorize with trie_codec let encoded_node = node_data; C::decode_compact_proof(encoded_node) .map_err(Error::DecodeError)? @@ -173,23 +172,19 @@ impl<'a, C: NodeCodecHybrid, H: BinaryHasher> StackEntry<'a, C, H> ), ChildProofHeader::Unused) } Node::Branch(_, _) => { - let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); - let register_children = &mut register_children[..]; C::branch_node_common( self.children.iter(), self.value, - register_children, // TODO unused register result + None, ) }, Node::NibbledBranch(partial, _, _) => { - let mut register_children: [Option<_>; NIBBLE_LENGTH] = Default::default(); - let register_children = &mut register_children[..]; C::branch_node_nibbled_common( partial.right_iter(), partial.len(), self.children.iter(), self.value, - register_children, // TODO again unused register result + None, ) }, }) @@ -487,7 +482,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve ChildReference::Hash(if let Some((bitmap_keys, additional_hash)) = last_entry.hybrid { let children = last_entry.children; let nb_children = children.iter().filter(|v| v.is_some()).count(); - let children = children.into_iter() + let children = children.iter() .enumerate() .filter_map(|(ix, v)| { v.as_ref().map(|v| (ix, v.clone())) diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 3031b1d9..4f7f8768 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -458,7 +458,7 @@ impl<'a, C: NodeCodecHybrid, F> DecoderStackEntry<'a, C, F> { C::branch_node_common( self.children.into_iter(), value, - register_children, + Some(register_children), ) } else { (C::branch_node( @@ -472,7 +472,7 @@ impl<'a, C: NodeCodecHybrid, F> DecoderStackEntry<'a, C, F> { partial.len(), self.children.iter(), value, - register_children, + Some(register_children), ) } else { (C::branch_node_nibbled( diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index e17ef059..35de60af 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -243,7 +243,7 @@ where // map the `NodeHandle`s from the Branch to `ChildReferences` children, value.as_ref().map(|v| &v[..]), - register_children, + Some(register_children), ) } else { (C::branch_node( @@ -273,7 +273,7 @@ where pr.len(), children, value.as_ref().map(|v| &v[..]), - register_children, + Some(register_children), ) } else { (C::branch_node_nibbled( From 9c3bf30489e287aa2bd34d18567a4df9add2c714 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 May 2020 18:58:05 +0200 Subject: [PATCH 49/73] change error --- trie-db/src/proof/verify.rs | 4 +--- trie-db/src/trie_codec.rs | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index a08d6f88..b942bb80 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -510,9 +510,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve ) { h } else { - // TODO better error for the invalid - // hybrid hash - return Err(Error::RootMismatch(Default::default())); + return Err(Error::DecodeError(L::Codec::codec_error("Invalid branch encoding for proof"))); } } else { L::Hash::hash(&node_data) diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 4f7f8768..12aec444 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -325,7 +325,6 @@ struct DecoderStackEntry<'a, C: NodeCodec, F> { /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The reconstructed child references. - /// TODO remove Vec here!!! children: Vec>>, /// Hybrid proof input hybrid: Option<(Bitmap, F)>, From abd126ef2820024fb1c3063de1f72b2b1ae6b262 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 15 May 2020 09:58:14 +0200 Subject: [PATCH 50/73] get rid of ordered trie hasher reference --- test-support/reference-trie/src/lib.rs | 10 ------ trie-db/src/node_codec.rs | 42 +------------------------- trie-db/src/proof/verify.rs | 3 +- 3 files changed, 2 insertions(+), 53 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index b3edb769..c5302ada 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -93,16 +93,6 @@ pub type RefLookup<'a, Q> = trie_db::Lookup<'a, ExtensionLayout, Q>; pub type RefLookupNoExt<'a, Q> = trie_db::Lookup<'a, NoExtensionLayout, Q>; -/// Typed version of hybrid_hash_node_adapter. -pub fn hybrid_hash_node_adapter_no_ext( - encoded_node: &[u8] -) -> std::result::Result::Out>, ()> { - trie_db::hybrid_hash_node_adapter::< - ReferenceNodeCodecNoExt, - RefHasher, - > (encoded_node) -} - pub fn reference_trie_root(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index e354afb5..b923e0e0 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -392,7 +392,7 @@ pub fn hybrid_hash_node_adapter, H range.as_hash(encoded_node) })); let mut buf = ::InnerHasher::init_buffer(); - Some(ordered_trie::OrderedTrieHasher::::hash_hybrid( + Some(Hasher::hash_hybrid( common.header(encoded_node), nb_children, children, @@ -406,43 +406,3 @@ pub fn hybrid_hash_node_adapter, H } ) } - -#[test] -fn test_hybrid_hash_node_adapter() { - use reference_trie::{RefTrieDBMutNoExt, TrieMut, hybrid_hash_node_adapter_no_ext, RefHasher}; - use crate::DBValue; - use memory_db::{MemoryDB, HashKey}; - use hash_db::EMPTY_PREFIX; - - - let mut db = MemoryDB::, DBValue>::default(); - let data = vec![ - (vec![0], vec![251; 34]), - (vec![0, 1], vec![252; 34]), - (vec![0, 1, 2], vec![255; 32]), - (vec![0, 1], vec![0; 38]), - ]; - let mut root = Default::default(); - { - let mut t = RefTrieDBMutNoExt::new(&mut db, &mut root); - for i in 0..data.len() { - let key: &[u8]= &data[i].0; - let value: &[u8] = &data[i].1; - t.insert(key, value).unwrap(); - } - } - let mut db2 = MemoryDB::, DBValue>::default(); - for (_key, (value, rc)) in db.clone().drain() { - if rc > 0 { - if !db2.insert_hybrid( - EMPTY_PREFIX, - &value[..], - hybrid_hash_node_adapter_no_ext, - ) { - panic!("invalid encoded node in proof"); - } - } - } - - assert!(db == db2); -} diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index b942bb80..4516e4a5 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -21,7 +21,6 @@ use crate::{ }; use hash_db::{Hasher, BinaryHasher, HasherHybrid}; use crate::node_codec::{Bitmap, HashesIter}; -use ordered_trie::OrderedTrieHasher; /// Errors that may occur during proof verification. Most of the errors types simply indicate that @@ -501,7 +500,7 @@ pub fn verify_proof<'a, L, I, K, V>(root: &::Out, proof: &[Ve } }); - if let Some(h) = OrderedTrieHasher::::InnerHasher>::hash_hybrid_proof( + if let Some(h) = L::Hash::hash_hybrid_proof( &common.header(node_data.as_slice())[..], nb_children, children, From 4fa465856275458f72fe9a4f10c57295531b3415 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 15 May 2020 11:51:42 +0200 Subject: [PATCH 51/73] removing more adapter unused code --- trie-db/src/lib.rs | 2 +- trie-db/src/node_codec.rs | 28 ---------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index a468873f..c07b5346 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -71,7 +71,7 @@ pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; pub use crate::node_codec::{NodeCodec, NodeCodecHybrid, Partial, HashDBHybridDyn, ChildProofHeader, - Bitmap, BITMAP_LENGTH, HashesPlan, hybrid_hash_node_adapter}; + Bitmap, BITMAP_LENGTH, HashesPlan}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybrid, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index b923e0e0..33862d09 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -378,31 +378,3 @@ impl Iterator for HashesPlan { (size, Some(size)) } } - -/// Adapter standard implementation to use with `HashDBInsertComplex` function. -pub fn hybrid_hash_node_adapter, Hasher: HasherHybrid>( - encoded_node: &[u8] -) -> crate::rstd::result::Result, ()> { - Codec::need_hybrid_proof(encoded_node).map(|hybrid| - if let Some((node, common)) = hybrid { - match node { - NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { - let nb_children = children.iter().filter(|v| v.is_some()).count(); - let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { - range.as_hash(encoded_node) - })); - let mut buf = ::InnerHasher::init_buffer(); - Some(Hasher::hash_hybrid( - common.header(encoded_node), - nb_children, - children, - &mut buf, - )) - }, - _ => unreachable!("hybrid only touch branch node"), - } - } else { - None - } - ) -} From 0ed528667c8852296a870384f2181e95c4446b00 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 15 May 2020 17:09:00 +0200 Subject: [PATCH 52/73] start switch test to generic use --- test-support/reference-trie/src/lib.rs | 195 ++++++++-------- trie-db/src/iter_build.rs | 70 +++--- trie-db/src/triedbmut.rs | 299 +++++++++++-------------- 3 files changed, 248 insertions(+), 316 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index c5302ada..99f2c9dc 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -27,6 +27,8 @@ use trie_db::{ DBValue, trie_visit, TrieBuilderHybrid, + TrieBuilder, + TrieRoot, TrieRootHybrid, Partial, HasherHybrid, @@ -56,22 +58,36 @@ pub type RefHasher = ordered_trie::OrderedTrieHasher; } impl TrieConfiguration for ExtensionLayout { } +/// Trie layout using extension nodes. +pub struct ExtensionLayoutHybrid; + +impl TrieLayout for ExtensionLayoutHybrid { + const USE_EXTENSION: bool = true; + const HYBRID_HASH: bool = true; + type Hash = RefHasher; + type Codec = ReferenceNodeCodec; +} + +impl TrieConfiguration for ExtensionLayoutHybrid { } + + /// Trie layout without extension nodes, allowing /// generic hasher. pub struct GenericNoExtensionLayout(PhantomData); impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; - const HYBRID_HASH: bool = true; + const HYBRID_HASH: bool = false; type Hash = H; type Codec = ReferenceNodeCodecNoExt; } @@ -81,16 +97,39 @@ impl TrieConfiguration for GenericNoExtensionLayout { } /// Trie layout without extension nodes. pub type NoExtensionLayout = GenericNoExtensionLayout; +/// Trie layout without extension nodes, allowing +/// generic hasher. +pub struct GenericNoExtensionLayoutHybrid(PhantomData); + +impl TrieLayout for GenericNoExtensionLayoutHybrid { + const USE_EXTENSION: bool = false; + const HYBRID_HASH: bool = true; + type Hash = H; + type Codec = ReferenceNodeCodecNoExt; +} + +impl TrieConfiguration for GenericNoExtensionLayoutHybrid { } + +/// Trie layout without extension nodes. +pub type NoExtensionLayoutHybrid = GenericNoExtensionLayoutHybrid; + + pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; +pub type RefTrieDBHybrid<'a> = trie_db::TrieDB<'a, ExtensionLayoutHybrid>; pub type RefTrieDBNoExt<'a> = trie_db::TrieDB<'a, NoExtensionLayout>; +pub type RefTrieDBNoExtHybrid<'a> = trie_db::TrieDB<'a, NoExtensionLayoutHybrid>; pub type RefTrieDBMut<'a> = trie_db::TrieDBMut<'a, ExtensionLayout>; +pub type RefTrieDBMutHybrid<'a> = trie_db::TrieDBMut<'a, ExtensionLayoutHybrid>; pub type RefTrieDBMutNoExt<'a> = trie_db::TrieDBMut<'a, NoExtensionLayout>; +pub type RefTrieDBMutNoExtHybrid<'a> = trie_db::TrieDBMut<'a, NoExtensionLayoutHybrid>; pub type RefFatDB<'a> = trie_db::FatDB<'a, ExtensionLayout>; pub type RefFatDBMut<'a> = trie_db::FatDBMut<'a, ExtensionLayout>; pub type RefSecTrieDB<'a> = trie_db::SecTrieDB<'a, ExtensionLayout>; pub type RefSecTrieDBMut<'a> = trie_db::SecTrieDBMut<'a, ExtensionLayout>; pub type RefLookup<'a, Q> = trie_db::Lookup<'a, ExtensionLayout, Q>; +pub type RefLookupHybrid<'a, Q> = trie_db::Lookup<'a, ExtensionLayoutHybrid, Q>; pub type RefLookupNoExt<'a, Q> = trie_db::Lookup<'a, NoExtensionLayout, Q>; +pub type RefLookupNoExtHybrid<'a, Q> = trie_db::Lookup<'a, NoExtensionLayoutHybrid, Q>; pub fn reference_trie_root(input: I) -> ::Out where @@ -136,44 +175,24 @@ fn data_sorted_unique(input: I) -> Vec<(A, B)> m.into_iter().collect() } -pub fn reference_trie_root_iter_build(input: I) -> ::Out where +pub fn reference_trie_root_iter_build(input: I) -> ::Out where + T: TrieLayout, I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = trie_db::TrieRootHybrid::::default(); - trie_visit::(data_sorted_unique(input), &mut cb); - cb.root.unwrap_or(Default::default()) -} - -pub fn reference_trie_root_no_extension_iter_build(input: I) -> ::Out where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - let mut cb = trie_db::TrieRootHybrid::::default(); - trie_visit::(data_sorted_unique(input), &mut cb); - cb.root.unwrap_or(Default::default()) -} - -fn reference_trie_root_unhashed_iter_build(input: I) -> Vec where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); - trie_visit::(data_sorted_unique(input), &mut cb); - cb.root.unwrap_or(Default::default()) -} - -fn reference_trie_root_unhashed_no_extension_iter_build(input: I) -> Vec where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); - trie_visit::(data_sorted_unique(input), &mut cb); - cb.root.unwrap_or(Default::default()) + match T::HYBRID_HASH { + true => { + let mut cb = trie_db::TrieRootHybrid::::default(); + trie_visit::(data_sorted_unique(input), &mut cb); + cb.root.unwrap_or(Default::default()) + }, + false => { + let mut cb = trie_db::TrieRoot::::default(); + trie_visit::(data_sorted_unique(input), &mut cb); + cb.root.unwrap_or(Default::default()) + }, + } } const EMPTY_TRIE: u8 = 0; @@ -1383,18 +1402,14 @@ pub fn compare_implementations + HashDBH } /// Compare trie builder and trie root implementations. -pub fn compare_root( +pub fn compare_root>( data: Vec<(Vec, Vec)>, - mut memdb: impl HashDBHybrid, + mut memdb: DB, ) { - let root_new = { - let mut cb = TrieRootHybrid::::default(); - trie_visit::(data.clone().into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) - }; + let root_new = reference_trie_root_iter_build::(data.clone()); let root = { let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } @@ -1404,78 +1419,48 @@ pub fn compare_root( assert_eq!(root, root_new); } -/// Compare trie builder and trie root unhashed implementations. -pub fn compare_unhashed( - data: Vec<(Vec, Vec)>, -) { - let root_new = { - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); - trie_visit::(data.clone().into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) - }; - let root = reference_trie_root_unhashed_iter_build(data); - - assert_eq!(root, root_new); -} - -/// Compare trie builder and trie root unhashed implementations. -/// This uses the variant without extension nodes. -pub fn compare_unhashed_no_extension( - data: Vec<(Vec, Vec)>, -) { - let root_new = { - let mut cb = trie_db::TrieRootUnhashedHybrid::::default(); - trie_visit::(data.clone().into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) - }; - let root = reference_trie_root_unhashed_no_extension_iter_build(data); - - assert_eq!(root, root_new); -} - -/// Trie builder root calculation utility. -pub fn calc_root( - data: I, -) -> ::Out - where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - let mut cb = TrieRootHybrid::::default(); - trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) -} - /// Trie builder root calculation utility. -/// This uses the variant without extension nodes. -pub fn calc_root_no_extension( +pub fn calc_root( data: I, -) -> ::Out +) -> ::Out where + T: TrieLayout, I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - let mut cb = TrieRootHybrid::::default(); - trie_db::trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + if T::HYBRID_HASH { + let mut cb = TrieRootHybrid::::default(); + trie_visit::(data.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } else { + let mut cb = TrieRoot::::default(); + trie_visit::(data.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } } /// Trie builder trie building utility. -pub fn calc_root_build( +pub fn calc_root_build( data: I, hashdb: &mut DB -) -> ::Out +) -> ::Out where + T: TrieLayout, I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + HashDBHybrid, + DB: hash_db::HashDB + HashDBHybrid, { - let mut cb = TrieBuilderHybrid::new(hashdb); - trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + if T::HYBRID_HASH { + let mut cb = TrieBuilderHybrid::new(hashdb); + trie_visit::(data.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } else { + let mut cb = TrieBuilder::new(hashdb); + trie_visit::(data.into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + } } /// Trie builder trie building utility. @@ -1585,21 +1570,21 @@ pub fn compare_implementations_no_extension_unordered( /// Testing utility that uses some periodic removal over /// its input test data. -pub fn compare_no_extension_insert_remove( +pub fn compare_insert_remove>( data: Vec<(bool, Vec, Vec)>, - mut memdb: impl HashDBHybrid, + mut memdb: DB, ) { let mut data2 = std::collections::BTreeMap::new(); let mut root = Default::default(); let mut a = 0; { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.commit(); } while a < data.len() { // new triemut every 3 element root = { - let mut t = RefTrieDBMutNoExt::from_existing(&mut memdb, &mut root).unwrap(); + let mut t = TrieDBMut::::from_existing(&mut memdb, &mut root).unwrap(); for _ in 0..3 { if data[a].0 { // remove @@ -1620,10 +1605,10 @@ pub fn compare_no_extension_insert_remove( *t.root() }; } - let mut t = RefTrieDBMutNoExt::from_existing(&mut memdb, &mut root).unwrap(); + let mut t = TrieDBMut::::from_existing(&mut memdb, &mut root).unwrap(); // we are testing the RefTrie code here so we do not sort or check uniqueness // before. - assert_eq!(*t.root(), calc_root_no_extension(data2)); + assert_eq!(*t.root(), calc_root::(data2)); } #[cfg(test)] diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index ada41f64..498a0fac 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -701,7 +701,20 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashe mod test { use crate::DBValue; use memory_db::{MemoryDB, HashKey, PrefixedKey}; - use reference_trie::RefHasher; + use reference_trie::{RefHasher, TrieLayout}; + use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; + + macro_rules! all_layout { + ($test:ident, $test_internal:ident) => { + #[test] + fn $test() { + $test_internal::(); + $test_internal::(); + $test_internal::(); + $test_internal::(); + } + }; + } #[test] fn trie_root_empty () { @@ -809,19 +822,13 @@ mod test { let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations_no_extension_unordered(data, memdb, hashdb); } - fn compare_no_extension_insert_remove(data: Vec<(bool, Vec, Vec)>) { + fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) { let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - reference_trie::compare_no_extension_insert_remove(data, memdb); - } - fn compare_root(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - reference_trie::compare_root(data, memdb); + reference_trie::compare_insert_remove::(data, memdb); } - fn compare_unhashed(data: Vec<(Vec, Vec)>) { - reference_trie::compare_unhashed(data); - } - fn compare_unhashed_no_extension(data: Vec<(Vec, Vec)>) { - reference_trie::compare_unhashed_no_extension(data); + fn compare_root(data: Vec<(Vec, Vec)>) { + let memdb = MemoryDB::, _>::default(); + reference_trie::compare_root::(data, memdb); } // Following tests are a bunch of detected issue here for non regression. @@ -843,34 +850,14 @@ mod test { (vec![1u8, 2u8, 3u8, 5u8, 3u8], vec![7u8;32]), ]); } - #[test] - fn root_extension_bis () { - compare_root(vec![ + all_layout!(root_extension_bis, root_extension_bis_internal); + fn root_extension_bis_internal() { + compare_root::(vec![ (vec![1u8, 2u8, 3u8, 3u8], vec![8u8;32]), (vec![1u8, 2u8, 3u8, 4u8], vec![7u8;32]), ]); } #[test] - fn root_extension_tierce () { - let d = vec![ - (vec![1u8, 2u8, 3u8, 3u8], vec![8u8;2]), - (vec![1u8, 2u8, 3u8, 4u8], vec![7u8;2]), - ]; - compare_unhashed(d.clone()); - compare_unhashed_no_extension(d); - } - #[test] - fn root_extension_tierce_big () { - // on more content unhashed would hash - compare_unhashed(vec![ - (vec![1u8, 2u8, 3u8, 3u8], vec![8u8;32]), - (vec![1u8, 2u8, 3u8, 4u8], vec![7u8;32]), - (vec![1u8, 6u8, 3u8, 3u8], vec![8u8;32]), - (vec![6u8, 2u8, 3u8, 3u8], vec![8u8;32]), - (vec![6u8, 2u8, 3u8, 13u8], vec![8u8;32]), - ]); - } - #[test] fn trie_middle_node2x () { compare_implementations(vec![ (vec![0u8, 2u8, 3u8, 5u8, 3u8], vec![1u8;2]), @@ -955,25 +942,25 @@ mod test { (vec![0x02, 0x50], vec![0x3]), ]); } - #[test] - fn fuzz_no_extension_insert_remove_1 () { + all_layout!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); + fn fuzz_no_extension_insert_remove_1_internal() { let data = vec![ (false, vec![0], vec![251, 255]), (false, vec![0, 1], vec![251, 255]), (false, vec![0, 1, 2], vec![255; 32]), (true, vec![0, 1], vec![0, 251]), ]; - compare_no_extension_insert_remove(data); + compare_insert_remove::(data); } - #[test] - fn fuzz_no_extension_insert_remove_2 () { + all_layout!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); + fn fuzz_no_extension_insert_remove_2_internal() { let data = vec![ (false, vec![0x00], vec![0xfd, 0xff]), (false, vec![0x10, 0x00], vec![1;32]), (false, vec![0x11, 0x10], vec![0;32]), (true, vec![0x10, 0x00], vec![]) ]; - compare_no_extension_insert_remove(data); + compare_insert_remove::(data); } #[test] fn two_bytes_nibble_length () { @@ -1005,5 +992,4 @@ mod test { (vec![105, 97, 48, 77, 101, 105, 121, 101], vec![69, 109, 111, 111, 82, 49, 97, 105]), ]); } - } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 35de60af..ab7e4ae6 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1690,38 +1690,32 @@ mod tests { use crate::DBValue; use memory_db::{MemoryDB, PrefixedKey}; use hash_db::Hasher; - use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, - ReferenceNodeCodec, reference_trie_root_iter_build as reference_trie_root, RefHasher, - reference_trie_root_no_extension_iter_build as reference_trie_root_no_extension}; + use reference_trie::{TrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, + ReferenceNodeCodec, ReferenceNodeCodecNoExt, reference_trie_root_iter_build as reference_trie_root, TrieLayout}; + use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; + use crate::nibble::BackingByteVec; - fn populate_trie<'db>( - db: &'db mut dyn HashDBHybridDyn, - root: &'db mut ::Out, - v: &[(Vec, Vec)] - ) -> RefTrieDBMut<'db> { - let mut t = RefTrieDBMut::new(db, root); - for i in 0..v.len() { - let key: &[u8]= &v[i].0; - let val: &[u8] = &v[i].1; - t.insert(key, val).unwrap(); - } - t - } - fn unpopulate_trie<'db>(t: &mut RefTrieDBMut<'db>, v: &[(Vec, Vec)]) { - for i in v { - let key: &[u8]= &i.0; - t.remove(key).unwrap(); - } + macro_rules! all_layout { + ($test:ident, $test_internal:ident) => { + #[test] + fn $test() { + $test_internal::(); + $test_internal::(); + $test_internal::(); + $test_internal::(); + } + }; } - fn populate_trie_no_extension<'db>( - db: &'db mut dyn HashDBHybridDyn, - root: &'db mut ::Out, + + fn populate_trie<'db, T: TrieLayout>( + db: &'db mut dyn HashDBHybridDyn, + root: &'db mut ::Out, v: &[(Vec, Vec)] - ) -> RefTrieDBMutNoExt<'db> { - let mut t = RefTrieDBMutNoExt::new(db, root); + ) -> TrieDBMut<'db, T> { + let mut t = TrieDBMut::::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -1730,20 +1724,30 @@ mod tests { t } - fn unpopulate_trie_no_extension<'db>(t: &mut RefTrieDBMutNoExt<'db>, v: &[(Vec, Vec)]) { + fn unpopulate_trie<'db, T: TrieLayout>(t: &mut TrieDBMut<'db, T>, v: &[(Vec, Vec)]) { for i in v { let key: &[u8]= &i.0; t.remove(key).unwrap(); } } - fn reference_hashed_null_node() -> ::Out { - as NodeCodec>::hashed_null_node() + fn reference_hashed_null_node() -> ::Out { + if T::USE_EXTENSION { + as NodeCodec>::hashed_null_node() + } else { + as NodeCodec>::hashed_null_node() + } } #[test] fn playpen() { env_logger::init(); + playpen_internal::(); + playpen_internal::(); + playpen_internal::(); + playpen_internal::(); + } + fn playpen_internal() { let mut seed = Default::default(); for test_i in 0..10 { if test_i % 50 == 0 { @@ -1757,10 +1761,10 @@ mod tests { count: 100, }.make_with(&mut seed); - let real = reference_trie_root(x.clone()); - let mut memdb = MemoryDB::, DBValue>::default(); + let real = reference_trie_root::(x.clone()); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); memtrie.commit(); if *memtrie.root() != real { @@ -1774,50 +1778,7 @@ mod tests { assert_eq!(*memtrie.root(), real); unpopulate_trie(&mut memtrie, &x); memtrie.commit(); - let hashed_null_node = reference_hashed_null_node(); - if *memtrie.root() != hashed_null_node { - println!("- TRIE MISMATCH"); - println!(""); - println!("{:#x?} vs {:#x?}", memtrie.root(), hashed_null_node); - for i in &x { - println!("{:#x?} -> {:#x?}", i.0, i.1); - } - } - assert_eq!(*memtrie.root(), hashed_null_node); - } - - // no_extension - let mut seed = Default::default(); - for test_i in 0..100 { - if test_i % 50 == 0 { - debug!("{:?} of 10000 stress tests done", test_i); - } - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 100, - }.make_with(&mut seed); - - let real = reference_trie_root_no_extension(x.clone()); - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - let mut memtrie = populate_trie_no_extension(&mut memdb, &mut root, &x); - - memtrie.commit(); - if *memtrie.root() != real { - println!("TRIE MISMATCH"); - println!(""); - println!("{:?} vs {:?}", memtrie.root(), real); - for i in &x { - println!("{:#x?} -> {:#x?}", i.0, i.1); - } - } - assert_eq!(*memtrie.root(), real); - unpopulate_trie_no_extension(&mut memtrie, &x); - memtrie.commit(); - let hashed_null_node = reference_hashed_null_node(); + let hashed_null_node = reference_hashed_null_node::(); if *memtrie.root() != hashed_null_node { println!("- TRIE MISMATCH"); println!(""); @@ -1830,34 +1791,34 @@ mod tests { } } - #[test] - fn init() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(init, init_internal); + fn init_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); - let hashed_null_node = reference_hashed_null_node(); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let hashed_null_node = reference_hashed_null_node::(); assert_eq!(*t.root(), hashed_null_node); } - #[test] - fn insert_on_empty() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_on_empty, insert_on_empty_internal); + fn insert_on_empty_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!( *t.root(), - reference_trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ]), + reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ]), ); } - #[test] - fn remove_to_empty() { + all_layout!(remove_to_empty, remove_to_empty_internal); + fn remove_to_empty_internal() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01], big_value).unwrap(); t.insert(&[0x01, 0x23], big_value).unwrap(); @@ -1865,8 +1826,8 @@ mod tests { t.remove(&[0x01]).unwrap(); } - #[test] - fn remove_to_empty_no_extension() { + all_layout!(remove_to_empty_no_extension, remove_to_empty_no_extension_internal); + fn remove_to_empty_no_extension_internal() { let big_value = b"00000000000000000000000000000000"; let big_value2 = b"00000000000000000000000000000002"; let big_value3 = b"00000000000000000000000000000004"; @@ -1874,7 +1835,7 @@ mod tests { let mut memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23], big_value3).unwrap(); t.insert(&[0x01], big_value2).unwrap(); @@ -1882,149 +1843,149 @@ mod tests { t.remove(&[0x01]).unwrap(); // commit on drop } - assert_eq!(&root[..], &reference_trie::calc_root_no_extension(vec![ + assert_eq!(&root, &reference_trie::calc_root::(vec![ (vec![0x01u8, 0x23], big_value3.to_vec()), (vec![0x01u8, 0x34], big_value.to_vec()), - ])[..]); + ])); } - #[test] - fn insert_replace_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_replace_root, insert_replace_root_internal); + fn insert_replace_root_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); assert_eq!( *t.root(), - reference_trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ]), + reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ]), ); } - #[test] - fn insert_make_branch_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_make_branch_root, insert_make_branch_root_internal); + fn insert_make_branch_root_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) ])); } - #[test] - fn insert_into_branch_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_into_branch_root, insert_into_branch_root_internal); + fn insert_into_branch_root_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), ])); } - #[test] - fn insert_value_into_branch_root() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_value_into_branch_root, insert_value_into_branch_root_internal); + fn insert_value_into_branch_root_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![], vec![0x0]), (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), ])); } - #[test] - fn insert_split_leaf() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_split_leaf, insert_split_leaf_internal); + fn insert_split_leaf_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), ])); } - #[test] - fn insert_split_extenstion() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(insert_split_extenstion, insert_split_extenstion_internal); + fn insert_split_extenstion_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![0x01, 0x23, 0x45], vec![0x01]), (vec![0x01, 0xf3, 0x45], vec![0x02]), (vec![0x01, 0xf3, 0xf5], vec![0x03]), ])); } - #[test] - fn insert_big_value() { + all_layout!(insert_big_value, insert_big_value_internal); + fn insert_big_value_internal() { let big_value0 = b"00000000000000000000000000000000"; let big_value1 = b"11111111111111111111111111111111"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![0x01u8, 0x23], big_value0.to_vec()), (vec![0x11u8, 0x23], big_value1.to_vec()) ])); } - #[test] - fn insert_duplicate_value() { + all_layout!(insert_duplicate_value, insert_duplicate_value_internal); + fn insert_duplicate_value_internal() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![ + assert_eq!(*t.root(), reference_trie_root::(vec![ (vec![0x01u8, 0x23], big_value.to_vec()), (vec![0x11u8, 0x23], big_value.to_vec()) ])); } - #[test] - fn test_at_empty() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(test_at_empty, test_at_empty_internal); + fn test_at_empty_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let t = RefTrieDBMut::new(&mut memdb, &mut root); + let t = TrieDBMut::::new(&mut memdb, &mut root); assert_eq!(t.get(&[0x5]).unwrap(), None); } - #[test] - fn test_at_one() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(test_at_one, test_at_one_internal); + fn test_at_one_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); t.commit(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); } - #[test] - fn test_at_three() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(test_at_three, test_at_three_internal); + fn test_at_three_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -2039,8 +2000,8 @@ mod tests { assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); } - #[test] - fn stress() { + all_layout!(stress, stress_internal); + fn stress_internal() { let mut seed = Default::default(); for _ in 0..50 { let x = StandardMap { @@ -2051,15 +2012,15 @@ mod tests { count: 4, }.make_with(&mut seed); - let real = reference_trie_root(x.clone()); - let mut memdb = MemoryDB::, DBValue>::default(); + let real = reference_trie_root::(x.clone()); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let mut memdb2 = MemoryDB::, DBValue>::default(); + let mut memdb2 = MemoryDB::, DBValue>::default(); let mut root2 = Default::default(); - let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); + let mut memtrie_sorted = populate_trie::(&mut memdb2, &mut root2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -2077,22 +2038,22 @@ mod tests { } } - #[test] - fn test_trie_existing() { - let mut db = MemoryDB::, DBValue>::default(); + all_layout!(test_trie_existing, test_trie_existing_internal); + fn test_trie_existing_internal() { + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::::new(&mut db, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } { - let _ = RefTrieDBMut::from_existing(&mut db, &mut root); + let _ = TrieDBMut::::from_existing(&mut db, &mut root); } } - #[test] - fn insert_empty() { + all_layout!(insert_empty, insert_empty_internal); + fn insert_empty_internal() { let mut seed = Default::default(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), @@ -2102,26 +2063,26 @@ mod tests { count: 4, }.make_with(&mut seed); - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::::new(&mut db, &mut root); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } - assert_eq!(*t.root(), reference_trie_root(x.clone())); + assert_eq!(*t.root(), reference_trie_root::(x.clone())); for &(ref key, _) in &x { t.insert(key, &[]).unwrap(); } assert!(t.is_empty()); - let hashed_null_node = reference_hashed_null_node(); + let hashed_null_node = reference_hashed_null_node::(); assert_eq!(*t.root(), hashed_null_node); } - #[test] - fn return_old_values() { + all_layout!(return_old_values, return_old_values_internal); + fn return_old_values_internal() { let mut seed = Default::default(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), @@ -2131,9 +2092,9 @@ mod tests { count: 2, }.make_with(&mut seed); - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::::new(&mut db, &mut root); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap().is_none()); assert_eq!(t.insert(key, value).unwrap(), Some(value.clone())); From fe0fed3516cd710e1671c36f3e3cc56b900f8c33 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 15 May 2020 17:19:44 +0200 Subject: [PATCH 53/73] fix fuzz --- trie-db/fuzz/Cargo.toml | 5 ---- trie-db/fuzz/fuzz_targets/trie_root.rs | 2 +- .../fuzz/fuzz_targets/trie_root_fix_len.rs | 2 +- .../fuzz/fuzz_targets/trie_unhashed_no_ext.rs | 10 ------- trie-db/fuzz/src/lib.rs | 29 ++++++++----------- 5 files changed, 14 insertions(+), 34 deletions(-) delete mode 100644 trie-db/fuzz/fuzz_targets/trie_unhashed_no_ext.rs diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index bff842f5..8674f4ab 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -28,11 +28,6 @@ members = ["."] name = "trie_root_new" path = "fuzz_targets/trie_root_new.rs" -[[bin]] -name = "trie_unhashed_no_ext" -path = "fuzz_targets/trie_unhashed_no_ext.rs" - - [[bin]] name = "trie_root" path = "fuzz_targets/trie_root.rs" diff --git a/trie-db/fuzz/fuzz_targets/trie_root.rs b/trie-db/fuzz/fuzz_targets/trie_root.rs index 32b65f83..75178bc2 100644 --- a/trie-db/fuzz/fuzz_targets/trie_root.rs +++ b/trie-db/fuzz/fuzz_targets/trie_root.rs @@ -5,5 +5,5 @@ use trie_db_fuzz::fuzz_that_reference_trie_root; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - fuzz_that_reference_trie_root(data); + fuzz_that_reference_trie_root::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs b/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs index 00763e26..902933a7 100644 --- a/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs +++ b/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs @@ -5,5 +5,5 @@ use trie_db_fuzz::fuzz_that_reference_trie_root_fix_length; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - fuzz_that_reference_trie_root_fix_length(data); + fuzz_that_reference_trie_root_fix_length::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/trie_unhashed_no_ext.rs b/trie-db/fuzz/fuzz_targets/trie_unhashed_no_ext.rs deleted file mode 100644 index dba4cc68..00000000 --- a/trie-db/fuzz/fuzz_targets/trie_unhashed_no_ext.rs +++ /dev/null @@ -1,10 +0,0 @@ - -#![no_main] - -use trie_db_fuzz::fuzz_that_unhashed_no_extension; -use libfuzzer_sys::fuzz_target; - -fuzz_target!(|data: &[u8]| { - // fuzzed code goes here - fuzz_that_unhashed_no_extension(data); -}); diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 374f803d..f21b8b2b 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -16,20 +16,20 @@ use hash_db::Hasher; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ - calc_root_no_extension, - compare_no_extension_insert_remove, + calc_root, ExtensionLayout, NoExtensionLayout, proof::{generate_proof, verify_proof}, reference_trie_root_iter_build as reference_trie_root, - RefTrieDBMut, + TrieDBMut, RefTrieDBMutNoExt, RefTrieDBNoExt, TrieDBIterator, RefHasher, + compare_insert_remove, }; use std::convert::TryInto; -use trie_db::{DBValue, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut}; +use trie_db::{DBValue, Trie, TrieDB, TrieLayout, TrieMut}; fn fuzz_to_data(input: &[u8]) -> Vec<(Vec,Vec)> { let mut result = Vec::new(); @@ -94,26 +94,26 @@ fn fuzz_removal(data: Vec<(Vec,Vec)>) -> Vec<(bool, Vec,Vec)> { res } -pub fn fuzz_that_reference_trie_root(input: &[u8]) { +pub fn fuzz_that_reference_trie_root(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } - assert_eq!(*t.root(), reference_trie_root(data)); + assert_eq!(*t.root(), reference_trie_root::(data)); } -pub fn fuzz_that_reference_trie_root_fix_length(input: &[u8]) { +pub fn fuzz_that_reference_trie_root_fix_length(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } - assert_eq!(*t.root(), reference_trie_root(data)); + assert_eq!(*t.root(), reference_trie_root::(data)); } fn fuzz_to_data_fix_length(input: &[u8]) -> Vec<(Vec,Vec)> { @@ -149,11 +149,6 @@ pub fn fuzz_that_compare_implementations(input: &[u8]) { reference_trie::compare_implementations(data, memdb, hashdb); } -pub fn fuzz_that_unhashed_no_extension(input: &[u8]) { - let data = data_sorted_unique(fuzz_to_data(input)); - reference_trie::compare_unhashed_no_extension(data); -} - pub fn fuzz_that_no_extension_insert(input: &[u8]) { let data = fuzz_to_data(input); //println!("data{:?}", data); @@ -167,7 +162,7 @@ pub fn fuzz_that_no_extension_insert(input: &[u8]) { // before. let data = data_sorted_unique(fuzz_to_data(input)); //println!("data{:?}", data); - assert_eq!(*t.root(), calc_root_no_extension(data)); + assert_eq!(*t.root(), calc_root::(data)); } pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { @@ -175,7 +170,7 @@ pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { let data = fuzz_removal(data); let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - compare_no_extension_insert_remove(data, memdb); + compare_insert_remove::(data, memdb); } pub fn fuzz_seek_iter(input: &[u8]) { From f7d54c8b1a484e299001157df0f84cbdab841d04 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 15 May 2020 18:38:33 +0200 Subject: [PATCH 54/73] continue test refactor, plus very important fix for iter_build --- test-support/reference-trie/src/lib.rs | 22 +++++++-------- trie-db/fuzz/fuzz_targets/trie_root.rs | 2 +- .../fuzz/fuzz_targets/trie_root_fix_len.rs | 2 +- trie-db/src/iter_build.rs | 27 ++++++++++++------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 99f2c9dc..34980d2b 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -1358,19 +1358,15 @@ impl NodeCodecHybrid for ReferenceNodeCodecNoExt { } /// Compare trie builder and in memory trie. -pub fn compare_implementations + HashDBHybrid + Eq> ( +pub fn compare_implementations + Eq> ( data: Vec<(Vec, Vec)>, mut memdb: X, mut hashdb: X, ) { - let root_new = { - let mut cb = TrieBuilderHybrid::new(&mut hashdb); - trie_visit::(data.clone().into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) - }; + let root_new = calc_root_build::(data.clone(), &mut hashdb); let root = { let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } @@ -1380,7 +1376,7 @@ pub fn compare_implementations + HashDBH if root_new != root { { let db : &dyn hash_db::HashDB<_, _> = &hashdb; - let t = RefTrieDB::new(&db, &root_new).unwrap(); + let t = TrieDB::::new(&db, &root_new).unwrap(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); @@ -1388,7 +1384,7 @@ pub fn compare_implementations + HashDBH } { let db : &dyn hash_db::HashDB<_, _> = &memdb; - let t = RefTrieDB::new(&db, &root).unwrap(); + let t = TrieDB::::new(&db, &root).unwrap(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); @@ -1450,7 +1446,7 @@ pub fn calc_root_build( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + HashDBHybrid, + DB: HashDBHybrid, { if T::HYBRID_HASH { let mut cb = TrieBuilderHybrid::new(hashdb); @@ -1473,7 +1469,7 @@ pub fn calc_root_build_no_extension( I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB + HashDBHybrid, + DB: HashDBHybrid, { let mut cb = TrieBuilderHybrid::new(hashdb); trie_db::trie_visit::(data.into_iter(), &mut cb); @@ -1527,8 +1523,8 @@ pub fn compare_implementations_no_extension( /// ordering before running when trie_build expect correct ordering). pub fn compare_implementations_no_extension_unordered( data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + HashDBHybrid, - mut hashdb: impl hash_db::HashDB + HashDBHybrid, + mut memdb: impl HashDBHybrid, + mut hashdb: impl HashDBHybrid, ) { let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { diff --git a/trie-db/fuzz/fuzz_targets/trie_root.rs b/trie-db/fuzz/fuzz_targets/trie_root.rs index 75178bc2..642c0566 100644 --- a/trie-db/fuzz/fuzz_targets/trie_root.rs +++ b/trie-db/fuzz/fuzz_targets/trie_root.rs @@ -5,5 +5,5 @@ use trie_db_fuzz::fuzz_that_reference_trie_root; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - fuzz_that_reference_trie_root::(data); + fuzz_that_reference_trie_root::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs b/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs index 902933a7..f7e5d0fd 100644 --- a/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs +++ b/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs @@ -5,5 +5,5 @@ use trie_db_fuzz::fuzz_that_reference_trie_root_fix_length; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - fuzz_that_reference_trie_root_fix_length::(data); + fuzz_that_reference_trie_root_fix_length::(data); }); diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 498a0fac..97a595f1 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -256,7 +256,7 @@ impl CacheAccum debug_assert!(self.0[last].2 == branch_d); // encode branch let v = self.0[last].1.take(); - let nkeyix = nkey.unwrap_or((0, 0)); + let nkeyix = nkey.unwrap_or((branch_d, 0)); let mut register_children = register_children_buf::(); let pr = NibbleSlice::new_offset(&key_branch, nkeyix.0); let encoded = if let Some(register_children) = register_children.as_mut() { @@ -273,11 +273,6 @@ impl CacheAccum self.0[last].0.as_ref().iter(), v.as_ref().map(|v| v.as_ref()), ), ChildProofHeader::Unused) }; - let ext_length = nkey.as_ref().map(|nkeyix| nkeyix.0).unwrap_or(0); - let pr = NibbleSlice::new_offset( - &key_branch, - branch_d - ext_length, - ); let result = if T::HYBRID_HASH { let len = self.0[last].0.as_ref().iter().filter(|v| v.is_some()).count(); let children = self.0[last].0.as_ref().iter(); @@ -798,14 +793,26 @@ mod test { } fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { + compare_implementations_prefixed_internal::(data.clone()); + compare_implementations_prefixed_internal::(data.clone()); + compare_implementations_prefixed_internal::(data.clone()); + compare_implementations_prefixed_internal::(data.clone()); + } + fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations(data, memdb, hashdb); + let hashdb = MemoryDB::, DBValue>::default(); + reference_trie::compare_implementations::(data, memdb, hashdb); } fn compare_implementations_h(data: Vec<(Vec, Vec)>) { + compare_implementations_h_internal::(data.clone()); + compare_implementations_h_internal::(data.clone()); + compare_implementations_h_internal::(data.clone()); + compare_implementations_h_internal::(data.clone()); + } + fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations(data, memdb, hashdb); + let hashdb = MemoryDB::, DBValue>::default(); + reference_trie::compare_implementations::(data.clone(), memdb, hashdb); } fn compare_implementations_no_extension(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); From 1b619fa954bd9d7aa7e42ed83a6d709ac2fe2d28 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sun, 17 May 2020 10:42:20 +0200 Subject: [PATCH 55/73] finish paremeterizing tests with trie layouts. --- test-support/reference-trie/src/lib.rs | 117 +----- trie-db/benches/bench.rs | 39 +- trie-db/fuzz/fuzz_targets/no_ext_insert.rs | 2 +- .../fuzz/fuzz_targets/no_ext_insert_rem.rs | 2 +- trie-db/fuzz/fuzz_targets/prefix_iter.rs | 2 +- trie-db/fuzz/fuzz_targets/seek_iter.rs | 2 +- trie-db/fuzz/fuzz_targets/trie_codec_proof.rs | 2 +- trie-db/fuzz/fuzz_targets/trie_proof_valid.rs | 4 +- trie-db/fuzz/fuzz_targets/trie_root_new.rs | 2 +- trie-db/fuzz/src/lib.rs | 46 +-- trie-db/src/iter_build.rs | 75 +--- trie-db/src/iterator.rs | 361 ++++++------------ trie-db/src/triedb.rs | 197 +++------- 13 files changed, 264 insertions(+), 587 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 34980d2b..7512806e 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -43,7 +43,7 @@ pub use trie_db::{ decode_compact, encode_compact, HashDBHybrid, HashDBHybridDyn, nibble_ops, NibbleSlice, NibbleVec, NodeCodec, proof, Record, Recorder, NodeCodecHybrid, Trie, TrieConfiguration, TrieDB, TrieDBIterator, TrieDBMut, TrieDBNodeIterator, TrieError, - TrieIterator, TrieLayout, TrieMut, Bitmap, BITMAP_LENGTH, + TrieIterator, TrieLayout, TrieMut, Bitmap, BITMAP_LENGTH, Lookup, }; pub use trie_root::TrieStream; pub mod node { @@ -115,13 +115,7 @@ pub type NoExtensionLayoutHybrid = GenericNoExtensionLayoutHybrid; pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; -pub type RefTrieDBHybrid<'a> = trie_db::TrieDB<'a, ExtensionLayoutHybrid>; -pub type RefTrieDBNoExt<'a> = trie_db::TrieDB<'a, NoExtensionLayout>; -pub type RefTrieDBNoExtHybrid<'a> = trie_db::TrieDB<'a, NoExtensionLayoutHybrid>; pub type RefTrieDBMut<'a> = trie_db::TrieDBMut<'a, ExtensionLayout>; -pub type RefTrieDBMutHybrid<'a> = trie_db::TrieDBMut<'a, ExtensionLayoutHybrid>; -pub type RefTrieDBMutNoExt<'a> = trie_db::TrieDBMut<'a, NoExtensionLayout>; -pub type RefTrieDBMutNoExtHybrid<'a> = trie_db::TrieDBMut<'a, NoExtensionLayoutHybrid>; pub type RefFatDB<'a> = trie_db::FatDB<'a, ExtensionLayout>; pub type RefFatDBMut<'a> = trie_db::FatDBMut<'a, ExtensionLayout>; pub type RefSecTrieDB<'a> = trie_db::SecTrieDB<'a, ExtensionLayout>; @@ -132,38 +126,21 @@ pub type RefLookupNoExt<'a, Q> = trie_db::Lookup<'a, NoExtensionLayout, Q>; pub type RefLookupNoExtHybrid<'a, Q> = trie_db::Lookup<'a, NoExtensionLayoutHybrid, Q>; -pub fn reference_trie_root(input: I) -> ::Out where +pub fn reference_trie_root(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, { - trie_root::trie_root::(input) -} - -/*fn reference_trie_root_unhashed(input: I) -> Vec where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - trie_root::unhashed_trie::(input) -}*/ - -pub fn reference_trie_root_no_extension(input: I) -> ::Out where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - trie_root::trie_root_no_extension::(input) + if T::HYBRID_HASH { + unimplemented!("trie_root does not implement hybrid hash, iter_build does") + } + if T::USE_EXTENSION { + trie_root::trie_root::(input) + } else { + trie_root::trie_root_no_extension::(input) + } } -/*fn reference_trie_root_unhashed_no_extension(input: I) -> Vec where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, -{ - trie_root::unhashed_trie_no_extension::(input) -}*/ - fn data_sorted_unique(input: I) -> Vec<(A, B)> where I: IntoIterator, @@ -1459,77 +1436,17 @@ pub fn calc_root_build( } } -/// Trie builder trie building utility. -/// This uses the variant without extension nodes. -pub fn calc_root_build_no_extension( - data: I, - hashdb: &mut DB, -) -> ::Out - where - I: IntoIterator, - A: AsRef<[u8]> + Ord + fmt::Debug, - B: AsRef<[u8]> + fmt::Debug, - DB: HashDBHybrid, -{ - let mut cb = TrieBuilderHybrid::new(hashdb); - trie_db::trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) -} - -/// Compare trie builder and in memory trie. -/// This uses the variant without extension nodes. -pub fn compare_implementations_no_extension( - data: Vec<(Vec, Vec)>, - mut memdb: impl hash_db::HashDB + HashDBHybrid, - mut hashdb: impl hash_db::HashDB + HashDBHybrid, -) { - let root_new = { - let mut cb = TrieBuilderHybrid::new(&mut hashdb); - trie_visit::(data.clone().into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) - }; - let root = { - let mut root = Default::default(); - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); - for i in 0..data.len() { - t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); - } - t.root().clone() - }; - - if root != root_new { - { - let db : &dyn hash_db::HashDB<_, _> = &memdb; - let t = RefTrieDBNoExt::new(&db, &root).unwrap(); - println!("{:?}", t); - for a in t.iter().unwrap() { - println!("a:{:?}", a); - } - } - { - let db : &dyn hash_db::HashDB<_, _> = &hashdb; - let t = RefTrieDBNoExt::new(&db, &root_new).unwrap(); - println!("{:?}", t); - for a in t.iter().unwrap() { - println!("a:{:?}", a); - } - } - } - - assert_eq!(root, root_new); -} - /// `compare_implementations_no_extension` for unordered input (trie_root does /// ordering before running when trie_build expect correct ordering). -pub fn compare_implementations_no_extension_unordered( +pub fn compare_implementations_unordered + Eq> ( data: Vec<(Vec, Vec)>, - mut memdb: impl HashDBHybrid, - mut hashdb: impl HashDBHybrid, + mut memdb: X, + mut hashdb: X, ) { let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { let mut root = Default::default(); - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); b_map.insert(data[i].0.clone(), data[i].1.clone()); @@ -1538,14 +1455,14 @@ pub fn compare_implementations_no_extension_unordered( }; let root_new = { let mut cb = TrieBuilderHybrid::new(&mut hashdb); - trie_visit::(b_map.into_iter(), &mut cb); + trie_visit::(b_map.into_iter(), &mut cb); cb.root.unwrap_or(Default::default()) }; if root != root_new { { let db : &dyn hash_db::HashDB<_, _> = &memdb; - let t = RefTrieDBNoExt::new(&db, &root).unwrap(); + let t = TrieDB::::new(&db, &root).unwrap(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); @@ -1553,7 +1470,7 @@ pub fn compare_implementations_no_extension_unordered( } { let db : &dyn hash_db::HashDB<_, _> = &hashdb; - let t = RefTrieDBNoExt::new(&db, &root_new).unwrap(); + let t = TrieDB::::new(&db, &root_new).unwrap(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); diff --git a/trie-db/benches/bench.rs b/trie-db/benches/bench.rs index 331b94e5..9f74d419 100644 --- a/trie-db/benches/bench.rs +++ b/trie-db/benches/bench.rs @@ -16,6 +16,7 @@ use criterion::{criterion_group, criterion_main, Bencher, black_box, Criterion}; use trie_db::{NibbleSlice, proof::{generate_proof, verify_proof}, Trie}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; +use reference_trie::ExtensionLayout as Layout; criterion_group!(benches, root_old, @@ -75,7 +76,7 @@ fn root_a_big_v(c: &mut Criterion) { .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data, ); @@ -96,7 +97,7 @@ fn root_b_big_v(c: &mut Criterion) { .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data, ); @@ -118,7 +119,7 @@ fn root_a_small_v(c: &mut Criterion) { .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data, ); @@ -139,7 +140,7 @@ fn root_b_small_v(c: &mut Criterion) { .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data, ); @@ -160,7 +161,7 @@ fn root_old(c: &mut Criterion) { .iter() .map(|v| (&v.0, &v.1)); - reference_trie::reference_trie_root(inputc); + reference_trie::reference_trie_root::(inputc); }), data, ); @@ -185,7 +186,7 @@ fn root_new(c: &mut Criterion) { .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data, ); @@ -287,7 +288,7 @@ fn trie_mut_root_a(c: &mut Criterion) { .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data); } @@ -306,7 +307,7 @@ fn trie_mut_root_b(c: &mut Criterion) { .map(|v| (&v.0, &v.1)) .collect::>(); - reference_trie::calc_root(inputc); + reference_trie::calc_root::(inputc); }), data); } @@ -326,7 +327,7 @@ fn trie_mut_ref_root_a(c: &mut Criterion) { .map(|v| (&v.0, &v.1)) .collect::>(); - reference_trie::reference_trie_root_iter_build(inputc); + reference_trie::reference_trie_root_iter_build::(inputc); }), data); } @@ -346,7 +347,7 @@ fn trie_mut_ref_root_b(c: &mut Criterion) { .map(|v| (&v.0, &v.1)) .collect::>(); - reference_trie::reference_trie_root_iter_build(inputc); + reference_trie::reference_trie_root_iter_build::(inputc); }), data); } @@ -366,7 +367,7 @@ fn trie_mut_a(c: &mut Criterion) { let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = reference_trie::RefTrieDBMut::new(&mut mdb, &mut root); + let mut trie = reference_trie::TrieDBMut::::new(&mut mdb, &mut root); for (key, value) in datac { trie.insert(&key, &value) .expect("changes trie: insertion to trie is not allowed to fail within runtime"); @@ -390,7 +391,7 @@ fn trie_mut_b(c: &mut Criterion) { let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = reference_trie::RefTrieDBMut::new(&mut mdb, &mut root); + let mut trie = reference_trie::TrieDBMut::::new(&mut mdb, &mut root); for (key, value) in datac { trie.insert(&key, &value) .expect("changes trie: insertion to trie is not allowed to fail within runtime"); @@ -416,7 +417,7 @@ fn trie_mut_build_a(c: &mut Criterion) { .collect::>(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - reference_trie::calc_root_build(inputc, &mut mdb); + reference_trie::calc_root_build::(inputc, &mut mdb); }), data); } @@ -438,7 +439,7 @@ fn trie_mut_build_b(c: &mut Criterion) { .collect::>(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - reference_trie::calc_root_build(inputc, &mut mdb); + reference_trie::calc_root_build::(inputc, &mut mdb); }), data); } @@ -449,11 +450,11 @@ fn trie_iteration(c: &mut Criterion) { let input = input2(29, 204800, 32); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let root = reference_trie::calc_root_build(input, &mut mdb); + let root = reference_trie::calc_root_build::(input, &mut mdb); c.bench_function("trie_iteration", move |b: &mut Bencher| b.iter(|| { - let trie = reference_trie::RefTrieDB::new(&mdb, &root).unwrap(); + let trie = reference_trie::TrieDB::::new(&mdb, &root).unwrap(); let mut iter = trie_db::TrieDBNodeIterator::new(&trie).unwrap(); assert!(iter.all(|result| result.is_ok())); }) @@ -475,9 +476,9 @@ fn trie_proof_verification(c: &mut Criterion) { keys.dedup(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let root = reference_trie::calc_root_build(data, &mut mdb); + let root = reference_trie::calc_root_build::(data, &mut mdb); - let trie = reference_trie::RefTrieDB::new(&mdb, &root).unwrap(); + let trie = reference_trie::TrieDB::::new(&mdb, &root).unwrap(); let proof = generate_proof(&trie, keys.iter()).unwrap(); let items = keys.into_iter() .map(|key| { @@ -488,7 +489,7 @@ fn trie_proof_verification(c: &mut Criterion) { c.bench_function("trie_proof_verification", move |b: &mut Bencher| b.iter(|| { - verify_proof::( + verify_proof::( &root, &proof, items.iter() diff --git a/trie-db/fuzz/fuzz_targets/no_ext_insert.rs b/trie-db/fuzz/fuzz_targets/no_ext_insert.rs index 4922e057..149d59da 100644 --- a/trie-db/fuzz/fuzz_targets/no_ext_insert.rs +++ b/trie-db/fuzz/fuzz_targets/no_ext_insert.rs @@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { // fuzzed code goes here - fuzz_that_no_extension_insert(data); + fuzz_that_no_extension_insert::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs b/trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs index 608e25fc..1b40103a 100644 --- a/trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs +++ b/trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs @@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { // fuzzed code goes here - fuzz_that_no_extension_insert_remove(data); + fuzz_that_no_extension_insert_remove::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/prefix_iter.rs b/trie-db/fuzz/fuzz_targets/prefix_iter.rs index 612989d2..3c2aa999 100644 --- a/trie-db/fuzz/fuzz_targets/prefix_iter.rs +++ b/trie-db/fuzz/fuzz_targets/prefix_iter.rs @@ -4,5 +4,5 @@ use trie_db_fuzz::fuzz_prefix_iter; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - fuzz_prefix_iter(data); + fuzz_prefix_iter::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/seek_iter.rs b/trie-db/fuzz/fuzz_targets/seek_iter.rs index f303b283..ef2116f4 100644 --- a/trie-db/fuzz/fuzz_targets/seek_iter.rs +++ b/trie-db/fuzz/fuzz_targets/seek_iter.rs @@ -4,5 +4,5 @@ use trie_db_fuzz::fuzz_seek_iter; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - fuzz_seek_iter(data); + fuzz_seek_iter::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs b/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs index 7e28ec89..ba7e92b6 100644 --- a/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs +++ b/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs @@ -4,5 +4,5 @@ use libfuzzer_sys::fuzz_target; use trie_db_fuzz::fuzz_that_trie_codec_proofs; fuzz_target!(|data: &[u8]| { - fuzz_that_trie_codec_proofs(data); + fuzz_that_trie_codec_proofs::(data); }); diff --git a/trie-db/fuzz/fuzz_targets/trie_proof_valid.rs b/trie-db/fuzz/fuzz_targets/trie_proof_valid.rs index 44f2b120..2c4141a0 100644 --- a/trie-db/fuzz/fuzz_targets/trie_proof_valid.rs +++ b/trie-db/fuzz/fuzz_targets/trie_proof_valid.rs @@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target; use trie_db_fuzz::fuzz_that_verify_accepts_valid_proofs; fuzz_target!(|data: &[u8]| { - fuzz_that_verify_accepts_valid_proofs(data); -}); \ No newline at end of file + fuzz_that_verify_accepts_valid_proofs::(data); +}); diff --git a/trie-db/fuzz/fuzz_targets/trie_root_new.rs b/trie-db/fuzz/fuzz_targets/trie_root_new.rs index 9e9d41fa..05b6e3a3 100644 --- a/trie-db/fuzz/fuzz_targets/trie_root_new.rs +++ b/trie-db/fuzz/fuzz_targets/trie_root_new.rs @@ -6,5 +6,5 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { // fuzzed code goes here - fuzz_that_compare_implementations(data); + fuzz_that_compare_implementations::(data); }); diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index f21b8b2b..36c0a183 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -18,14 +18,10 @@ use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ calc_root, ExtensionLayout, - NoExtensionLayout, proof::{generate_proof, verify_proof}, reference_trie_root_iter_build as reference_trie_root, TrieDBMut, - RefTrieDBMutNoExt, - RefTrieDBNoExt, TrieDBIterator, - RefHasher, compare_insert_remove, }; use std::convert::TryInto; @@ -141,20 +137,20 @@ fn data_sorted_unique(input: Vec<(Vec,Vec)>) -> Vec<(Vec,Vec)> { m.into_iter().collect() } -pub fn fuzz_that_compare_implementations(input: &[u8]) { +pub fn fuzz_that_compare_implementations(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data(input)); //println!("data:{:?}", &data); let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations(data, memdb, hashdb); + let hashdb = MemoryDB::, DBValue>::default(); + reference_trie::compare_implementations::(data, memdb, hashdb); } -pub fn fuzz_that_no_extension_insert(input: &[u8]) { +pub fn fuzz_that_no_extension_insert(input: &[u8]) { let data = fuzz_to_data(input); //println!("data{:?}", data); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } @@ -162,24 +158,24 @@ pub fn fuzz_that_no_extension_insert(input: &[u8]) { // before. let data = data_sorted_unique(fuzz_to_data(input)); //println!("data{:?}", data); - assert_eq!(*t.root(), calc_root::(data)); + assert_eq!(*t.root(), calc_root::(data)); } -pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { +pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { let data = fuzz_to_data(input); let data = fuzz_removal(data); let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - compare_insert_remove::(data, memdb); + compare_insert_remove::(data, memdb); } -pub fn fuzz_seek_iter(input: &[u8]) { +pub fn fuzz_seek_iter(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } @@ -198,7 +194,7 @@ pub fn fuzz_seek_iter(input: &[u8]) { let mut iter_res = Vec::new(); let mut error = 0; { - let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap(); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = trie.iter().unwrap(); if let Ok(_) = iter.seek(prefix) { } else { @@ -222,13 +218,13 @@ pub fn fuzz_seek_iter(input: &[u8]) { assert_eq!(error, 0); } -pub fn fuzz_prefix_iter(input: &[u8]) { +pub fn fuzz_prefix_iter(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } @@ -247,7 +243,7 @@ pub fn fuzz_prefix_iter(input: &[u8]) { let mut iter_res = Vec::new(); let mut error = 0; { - let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap(); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let iter = TrieDBIterator::new_prefixed(&trie, prefix).unwrap(); for x in iter { @@ -268,7 +264,7 @@ pub fn fuzz_prefix_iter(input: &[u8]) { assert_eq!(error, 0); } -pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) { +pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) { let mut data = fuzz_to_data(input); // Split data into 3 parts: // - the first 1/3 is added to the trie and not included in the proof @@ -284,11 +280,11 @@ pub fn fuzz_that_verify_accepts_valid_proofs(input: &[u8]) { keys.sort(); keys.dedup(); - let (root, proof, items) = test_generate_proof::(data, keys); - assert!(verify_proof::(&root, &proof, items.iter()).is_ok()); + let (root, proof, items) = test_generate_proof::(data, keys); + assert!(verify_proof::(&root, &proof, items.iter()).is_ok()); } -pub fn fuzz_that_trie_codec_proofs(input: &[u8]) { +pub fn fuzz_that_trie_codec_proofs(input: &[u8]) { let mut data = fuzz_to_data(input); // Split data into 3 parts: // - the first 1/3 is added to the trie and not included in the proof @@ -304,7 +300,7 @@ pub fn fuzz_that_trie_codec_proofs(input: &[u8]) { keys.sort(); keys.dedup(); - test_trie_codec_proof::(data, keys); + test_trie_codec_proof::(data, keys); } pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) { @@ -446,6 +442,6 @@ fn test_trie_codec_proof( fn debug_error() { let input = [0x0,0x0,0x0,0xcd,0xf8,0xff,0xff,0xff,0x0,0xcd,0x2]; // let input = [0x40,0x0,0xe6,0xff,0xff,0x40,0x0,0xe6,0xff,0xff,0x2b]; - fuzz_that_verify_accepts_valid_proofs(&input[..]); - fuzz_that_trie_codec_proofs(&input[..]); + fuzz_that_verify_accepts_valid_proofs::(&input[..]); + fuzz_that_trie_codec_proofs::(&input[..]); } diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 97a595f1..1a69c6fb 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -731,46 +731,20 @@ mod test { ]); } - fn test_iter(data: Vec<(Vec, Vec)>) { - use reference_trie::{RefTrieDBMut, TrieMut, RefTrieDB, Trie}; + fn test_iter(data: Vec<(Vec, Vec)>) { + use reference_trie::{TrieMut, Trie, TrieDBMut, TrieDB}; - let mut db = MemoryDB::, DBValue>::default(); + let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::::new(&mut db, &mut root); for i in 0..data.len() { let key: &[u8]= &data[i].0; let value: &[u8] = &data[i].1; t.insert(key, value).unwrap(); } } - let t = RefTrieDB::new(&db, &root).unwrap(); - for (i, kv) in t.iter().unwrap().enumerate() { - let (k, v) = kv.unwrap(); - let key: &[u8]= &data[i].0; - let value: &[u8] = &data[i].1; - assert_eq!(k, key); - assert_eq!(v, value); - } - for (k, v) in data.into_iter() { - assert_eq!(&t.get(&k[..]).unwrap().unwrap()[..], &v[..]); - } - } - - fn test_iter_no_extension(data: Vec<(Vec, Vec)>) { - use reference_trie::{RefTrieDBMutNoExt, TrieMut, RefTrieDBNoExt, Trie}; - - let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMutNoExt::new(&mut db, &mut root); - for i in 0..data.len() { - let key: &[u8]= &data[i].0; - let value: &[u8] = &data[i].1; - t.insert(key, value).unwrap(); - } - } - let t = RefTrieDBNoExt::new(&db, &root).unwrap(); + let t = TrieDB::::new(&db, &root).unwrap(); for (i, kv) in t.iter().unwrap().enumerate() { let (k, v) = kv.unwrap(); let key: &[u8]= &data[i].0; @@ -784,12 +758,12 @@ mod test { } fn compare_implementations(data: Vec<(Vec, Vec)>) { - test_iter(data.clone()); - test_iter_no_extension(data.clone()); + test_iter::(data.clone()); + test_iter::(data.clone()); + test_iter::(data.clone()); + test_iter::(data.clone()); compare_implementations_h(data.clone()); compare_implementations_prefixed(data.clone()); - compare_implementations_no_extension(data.clone()); - compare_implementations_no_extension_prefixed(data.clone()); } fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { @@ -814,20 +788,10 @@ mod test { let hashdb = MemoryDB::, DBValue>::default(); reference_trie::compare_implementations::(data.clone(), memdb, hashdb); } - fn compare_implementations_no_extension(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations_no_extension(data, memdb, hashdb); - } - fn compare_implementations_no_extension_prefixed(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations_no_extension(data, memdb, hashdb); - } fn compare_implementations_no_extension_unordered(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations_no_extension_unordered(data, memdb, hashdb); + reference_trie::compare_implementations_unordered::(data, memdb, hashdb); } fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) { let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); @@ -943,7 +907,7 @@ mod test { } #[test] fn fuzz_no_extension4 () { - compare_implementations_no_extension(vec![ + compare_implementations(vec![ (vec![0x01, 0x56], vec![0x1]), (vec![0x02, 0x42], vec![0x2]), (vec![0x02, 0x50], vec![0x3]), @@ -975,21 +939,24 @@ mod test { (vec![00u8], vec![0]), (vec![01u8;64], vec![0;32]), ]; - compare_implementations_no_extension(data.clone()); - compare_implementations_no_extension_prefixed(data.clone()); + compare_implementations_prefixed(data.clone()); } #[test] #[should_panic] fn too_big_nibble_length_old () { - compare_implementations_h(vec![ - (vec![01u8;64], vec![0;32]), - ]); + compare_implementations_prefixed_internal::( + vec![(vec![01u8;64], vec![0;32])], + ); } #[test] fn too_big_nibble_length_new () { - compare_implementations_no_extension(vec![ + // this is valid for no_ext code only, + // the other one got maximal length in encoding. + let data = vec![ (vec![01u8;((u16::max_value() as usize + 1) / 2) + 1], vec![0;32]), - ]); + ]; + compare_implementations_prefixed_internal::(data.clone()); + compare_implementations_prefixed_internal::(data.clone()); } #[test] fn polka_re_test () { diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 2e1b5ec1..f1ceb7c4 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -389,35 +389,33 @@ mod tests { use hex_literal::hex; use hash_db::{HashDB, Hasher}; use reference_trie::{ - RefTrieDB, RefTrieDBMut, + TrieDB, TrieDBMut, TrieError, TrieMut, TrieIterator, TrieDBNodeIterator, NibbleSlice, NibbleVec, - node::Node, RefHasher, + node::Node, TrieLayout, }; - use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt}; - - type MemoryDB = memory_db::MemoryDB, DBValue>; - - fn build_trie_db_with_extension(pairs: &[(Vec, Vec)]) - -> (MemoryDB, ::Out) - { - let mut memdb = MemoryDB::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); - for (x, y) in pairs.iter() { - t.insert(x, y).unwrap(); + use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; + + macro_rules! all_layout { + ($test:ident, $test_internal:ident) => { + #[test] + fn $test() { + $test_internal::(); + $test_internal::(); + $test_internal::(); + $test_internal::(); } - } - (memdb, root) + }; } - fn build_trie_db_without_extension(pairs: &[(Vec, Vec)]) - -> (MemoryDB, ::Out) + type MemoryDB = memory_db::MemoryDB<::Hash, memory_db::PrefixedKey<::Hash>, DBValue>; + + fn build_trie_db(pairs: &[(Vec, Vec)]) + -> (MemoryDB, ::Out) { - let mut memdb = MemoryDB::default(); + let mut memdb = MemoryDB::::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in pairs.iter() { t.insert(x, y).unwrap(); } @@ -435,113 +433,76 @@ mod tests { v } - #[test] - fn iterator_works_with_extension() { + all_layout!(iterator_works, iterator_works_internal); + fn iterator_works_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db_with_extension(&pairs); - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + let (memdb, root) = build_trie_db::(&pairs); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - match iter.next() { - Some(Ok((prefix, Some(_), node))) => { - assert_eq!(prefix, nibble_vec(hex!(""), 0)); - match node.node() { - Node::Extension(partial, _) => - assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), - _ => panic!("unexpected node"), - } - } - _ => panic!("unexpected item"), - } - - match iter.next() { - Some(Ok((prefix, Some(_), node))) => { - assert_eq!(prefix, nibble_vec(hex!("00"), 1)); - match node.node() { - Node::Branch(_, _) => {}, - _ => panic!("unexpected node"), + if T::USE_EXTENSION { + match iter.next() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::Extension(partial, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), - } - match iter.next() { - Some(Ok((prefix, None, node))) => { - assert_eq!(prefix, nibble_vec(hex!("01"), 2)); - match node.node() { - Node::Branch(_, _) => {}, - _ => panic!("unexpected node"), + match iter.next() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!("00"), 1)); + match node.node() { + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), - } - match iter.next() { - Some(Ok((prefix, None, node))) => { - assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); - match node.node() { - Node::Leaf(partial, _) => - assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), - _ => panic!("unexpected node"), - } - } - _ => panic!("unexpected item"), - } - - match iter.next() { - Some(Ok((prefix, Some(_), node))) => { - assert_eq!(prefix, nibble_vec(hex!("02"), 2)); - match node.node() { - Node::Leaf(partial, _) => - assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), - _ => panic!("unexpected node"), + match iter.next() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), - } - - assert!(iter.next().is_none()); - } - - #[test] - fn iterator_works_without_extension() { - let pairs = vec![ - (hex!("01").to_vec(), b"aaaa".to_vec()), - (hex!("0123").to_vec(), b"bbbb".to_vec()), - (hex!("02").to_vec(), vec![1; 32]), - ]; - - let (memdb, root) = build_trie_db_without_extension(&pairs); - let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap(); - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - - match iter.next() { - Some(Ok((prefix, Some(_), node))) => { - assert_eq!(prefix, nibble_vec(hex!(""), 0)); - match node.node() { - Node::NibbledBranch(partial, _, _) => - assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), - _ => panic!("unexpected node"), + } else { + match iter.next() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), - } - match iter.next() { - Some(Ok((prefix, None, node))) => { - assert_eq!(prefix, nibble_vec(hex!("01"), 2)); - match node.node() { - Node::NibbledBranch(partial, _, _) => - assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), - _ => panic!("unexpected node"), + match iter.next() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), } match iter.next() { @@ -553,7 +514,6 @@ mod tests { _ => panic!("unexpected node"), } } - _ => panic!("unexpected item"), } @@ -572,10 +532,10 @@ mod tests { assert!(iter.next().is_none()); } - #[test] - fn iterator_over_empty_works() { - let (memdb, root) = build_trie_db_with_extension(&[]); - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + all_layout!(iterator_over_empty_works, iterator_over_empty_works_internal); + fn iterator_over_empty_works_internal() { + let (memdb, root) = build_trie_db::(&[]); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); match iter.next() { @@ -592,16 +552,16 @@ mod tests { assert!(iter.next().is_none()); } - #[test] - fn seek_works_with_extension() { + all_layout!(seek_works, seek_works_internal); + fn seek_works_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db_with_extension(&pairs); - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + let (memdb, root) = build_trie_db::(&pairs); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("")[..]).unwrap(); @@ -636,55 +596,10 @@ mod tests { assert!(iter.next().is_none()); } - - #[test] - fn seek_works_without_extension() { - let pairs = vec![ - (hex!("01").to_vec(), b"aaaa".to_vec()), - (hex!("0123").to_vec(), b"bbbb".to_vec()), - (hex!("02").to_vec(), vec![1; 32]), - ]; - - let (memdb, root) = build_trie_db_without_extension(&pairs); - let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap(); - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - - TrieIterator::seek(&mut iter, &hex!("")[..]).unwrap(); - match iter.next() { - Some(Ok((prefix, _, _))) => - assert_eq!(prefix, nibble_vec(hex!(""), 0)), - _ => panic!("unexpected item"), - } - - TrieIterator::seek(&mut iter, &hex!("00")[..]).unwrap(); - match iter.next() { - Some(Ok((prefix, _, _))) => - assert_eq!(prefix, nibble_vec(hex!("01"), 2)), - _ => panic!("unexpected item"), - } - - TrieIterator::seek(&mut iter, &hex!("01")[..]).unwrap(); - match iter.next() { - Some(Ok((prefix, _, _))) => - assert_eq!(prefix, nibble_vec(hex!("01"), 2)), - _ => panic!("unexpected item"), - } - - TrieIterator::seek(&mut iter, &hex!("02")[..]).unwrap(); - match iter.next() { - Some(Ok((prefix, _, _))) => - assert_eq!(prefix, nibble_vec(hex!("02"), 2)), - _ => panic!("unexpected item"), - } - - TrieIterator::seek(&mut iter, &hex!("03")[..]).unwrap(); - assert!(iter.next().is_none()); - } - - #[test] - fn seek_over_empty_works() { - let (memdb, root) = build_trie_db_with_extension(&[]); - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + all_layout!(seek_over_empty_works, seek_over_empty_works_internal); + fn seek_over_empty_works_internal() { + let (memdb, root) = build_trie_db::(&[]); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("")[..]).unwrap(); @@ -703,8 +618,8 @@ mod tests { assert!(iter.next().is_none()); } - #[test] - fn iterate_over_incomplete_db() { + all_layout!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); + fn iterate_over_incomplete_db_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), @@ -712,11 +627,11 @@ mod tests { (hex!("03").to_vec(), vec![2; 32]), ]; - let (mut memdb, root) = build_trie_db_with_extension(&pairs); + let (mut memdb, root) = build_trie_db::(&pairs); // Look up the leaf node with prefix "02". let leaf_hash = { - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("02")[..]).unwrap(); @@ -737,23 +652,32 @@ mod tests { // Seek to missing node returns error. { - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); match TrieIterator::seek(&mut iter, &hex!("02")[..]) { - Err(ref err) if **err == TrieError::IncompleteDatabase(leaf_hash) => {}, + Err(e) => { + if let TrieError::IncompleteDatabase(err_hash) = *e { + assert_eq!(err_hash.as_ref(), leaf_hash.as_ref()); + } + }, _ => panic!("expected IncompleteDatabase error"), } } // Iterate over missing node works. { - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("0130")[..]).unwrap(); match iter.next() { - Some(Err(ref err)) if **err == TrieError::IncompleteDatabase(leaf_hash) => {}, + Some(Err(e)) => { + if let TrieError::IncompleteDatabase(err_hash) = *e { + assert_eq!(err_hash.as_ref(), leaf_hash.as_ref()); + } + }, + _ => panic!("expected IncompleteDatabase error"), } match iter.next() { @@ -771,78 +695,43 @@ mod tests { } } - #[test] - fn prefix_works_with_extension() { + all_layout!(prefix_works, prefix_works_internal); + fn prefix_works_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db_with_extension(&pairs); - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + let (memdb, root) = build_trie_db::(&pairs); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("01").to_vec()[..]).unwrap(); - match iter.next() { - Some(Ok((prefix, None, node))) => { - assert_eq!(prefix, nibble_vec(hex!("01"), 2)); - match node.node() { - Node::Branch(_, _) => {}, - _ => panic!("unexpected node"), - } - } - _ => panic!("unexpected item"), - } - - match iter.next() { - Some(Ok((prefix, None, node))) => { - assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); - match node.node() { - Node::Leaf(partial, _) => - assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), - _ => panic!("unexpected node"), + if T::USE_EXTENSION { + match iter.next() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), - } - - assert!(iter.next().is_none()); - - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - iter.prefix(&hex!("0010").to_vec()[..]).unwrap(); - assert!(iter.next().is_none()); - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - iter.prefix(&hex!("10").to_vec()[..]).unwrap(); - assert!(iter.next().is_none()); - - } - - #[test] - fn prefix_works_without_extension() { - let pairs = vec![ - (hex!("01").to_vec(), b"aaaa".to_vec()), - (hex!("0123").to_vec(), b"bbbb".to_vec()), - (hex!("02").to_vec(), vec![1; 32]), - ]; - - let (memdb, root) = build_trie_db_without_extension(&pairs); - let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap(); - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - - iter.prefix(&hex!("01").to_vec()[..]).unwrap(); - - match iter.next() { - Some(Ok((prefix, None, node))) => { - assert_eq!(prefix, nibble_vec(hex!("01"), 2)); - match node.node() { - Node::NibbledBranch(partial, _, _) => - assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), - _ => panic!("unexpected node"), + } else { + match iter.next() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), + _ => panic!("unexpected node"), + } } + _ => panic!("unexpected item"), } - _ => panic!("unexpected item"), } match iter.next() { @@ -865,13 +754,12 @@ mod tests { let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("10").to_vec()[..]).unwrap(); assert!(iter.next().is_none()); - } - #[test] - fn prefix_over_empty_works() { - let (memdb, root) = build_trie_db_with_extension(&[]); - let trie = RefTrieDB::new(&memdb, &root).unwrap(); + all_layout!(prefix_over_empty_works, prefix_over_empty_works_internal); + fn prefix_over_empty_works_internal() { + let (memdb, root) = build_trie_db::(&[]); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("")[..]).unwrap(); match iter.next() { @@ -892,4 +780,3 @@ mod tests { assert!(iter.next().is_none()); } } - diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index cddaa538..51f2f28a 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -332,55 +332,39 @@ impl<'a, L: TrieLayout> Iterator for TrieDBIterator<'a, L> { mod tests { use memory_db::{MemoryDB, PrefixedKey}; use crate::DBValue; - use reference_trie::{RefTrieDB, RefTrieDBMut, RefLookup, Trie, TrieMut, NibbleSlice}; - use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt, RefHasher}; + use reference_trie::{TrieDB, TrieDBMut, Lookup, Trie, TrieMut, NibbleSlice, TrieLayout}; + use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; use hex_literal::hex; - #[test] - fn iterator_works() { - let pairs = vec![ - (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), - (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), - ]; - - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); + macro_rules! all_layout { + ($test:ident, $test_internal:ident) => { + #[test] + fn $test() { + $test_internal::(); + $test_internal::(); + $test_internal::(); + $test_internal::(); } - } - - let trie = RefTrieDB::new(&memdb, &root).unwrap(); - - let iter = trie.iter().unwrap(); - let mut iter_pairs = Vec::new(); - for pair in iter { - let (key, value) = pair.unwrap(); - iter_pairs.push((key, value.to_vec())); - } - - assert_eq!(pairs, iter_pairs); + }; } - #[test] - fn iterator_works_without_extension() { + all_layout!(iterator_works, iterator_works_internal); + fn iterator_works_internal() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in &pairs { t.insert(x, y).unwrap(); } } - let trie = RefTrieDBNoExt::new(&memdb, &root).unwrap(); + let trie = TrieDB::::new(&memdb, &root).unwrap(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); @@ -392,23 +376,23 @@ mod tests { assert_eq!(pairs, iter_pairs); } - #[test] - fn iterator_seek_works() { + all_layout!(iterator_seek_works, iterator_seek_works_internal); + fn iterator_seek_works_internal() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in &pairs { t.insert(x, y).unwrap(); } } - let t = RefTrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = t.iter().unwrap(); assert_eq!( @@ -435,44 +419,8 @@ mod tests { ); } - #[test] - fn iterator_seek_works_without_extension() { - let pairs = vec![ - (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), - (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), - ]; - - let mut memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); - } - } - - let t = RefTrieDBNoExt::new(&memdb, &root).unwrap(); - - let mut iter = t.iter().unwrap(); - assert_eq!( - iter.next().unwrap().unwrap(), - (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()) - ); - iter.seek(&hex!("00")[..]).unwrap(); - assert_eq!( - pairs, - iter.map(|x| x.unwrap()).map(|(k, v)| (k, v[..].to_vec())).collect::>(), - ); - let mut iter = t.iter().unwrap(); - iter.seek(&hex!("0103000000000000000465")[..]).unwrap(); - assert_eq!( - &pairs[1..], - &iter.map(|x| x.unwrap()).map(|(k, v)| (k, v[..].to_vec())).collect::>()[..], - ); - } - - #[test] - fn iterator() { + all_layout!(iterator, iterator_internal); + fn iterator_internal() { let d = vec![ b"A".to_vec(), b"AA".to_vec(), @@ -480,16 +428,16 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for x in &d { t.insert(x, x).unwrap(); } } - let t = RefTrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::::new(&memdb, &root).unwrap(); assert_eq!( d.iter() .map(|i| i.clone()) @@ -502,8 +450,8 @@ mod tests { assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::>()); } - #[test] - fn iterator_without_extension() { + all_layout!(iterator_seek, iterator_seek_internal); + fn iterator_seek_internal() { let d = vec![ b"A".to_vec(), b"AA".to_vec(), @@ -511,42 +459,16 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for x in &d { t.insert(x, x).unwrap(); } } - let t = RefTrieDBNoExt::new(&memdb, &root).unwrap(); - assert_eq!( - d.iter().map(|i| i.clone()).collect::>(), - t.iter().unwrap().map(|x| x.unwrap().0).collect::>(), - ); - assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::>()); - } - - #[test] - fn iterator_seek() { - let d = vec![ - b"A".to_vec(), - b"AA".to_vec(), - b"AB".to_vec(), - b"B".to_vec(), - ]; - - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); - for x in &d { - t.insert(x, x).unwrap(); - } - } - - let t = RefTrieDBNoExt::new(&memdb, &root).unwrap(); + let t = TrieDB::::new(&memdb, &root).unwrap(); let mut iter = t.iter().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (b"A".to_vec(), b"A".to_vec())); iter.seek(b"!").unwrap(); @@ -574,40 +496,24 @@ mod tests { assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } - #[test] - fn get_length_with_extension() { - let mut memdb = MemoryDB::, DBValue>::default(); + all_layout!(get_length_with_extension, get_length_with_extension_internal); + fn get_length_with_extension_internal() { + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); } - let t = RefTrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::::new(&memdb, &root).unwrap(); assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()).unwrap(), Some(32)); assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()).unwrap(), None); } - #[test] - fn get_length_without_extension() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMutNoExt::new(&mut memdb, &mut root); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBA").unwrap(); - } - - let t = RefTrieDBNoExt::new(&memdb, &root).unwrap(); - assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); - assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()).unwrap(), Some(5)); - assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()).unwrap(), None); - } - - #[test] - fn debug_output_supports_pretty_print() { + all_layout!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); + fn debug_output_supports_pretty_print_internal() { let d = vec![ b"A".to_vec(), b"AA".to_vec(), @@ -615,18 +521,19 @@ mod tests { b"B".to_vec(), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); let root = { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for x in &d { t.insert(x, x).unwrap(); } t.root().clone() }; - let t = RefTrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::::new(&memdb, &root).unwrap(); - assert_eq!(format!("{:#?}", t), + if T::USE_EXTENSION { + assert_eq!(format!("{:#?}", t), "TrieDB { hash_count: 0, root: Node::Extension { @@ -676,26 +583,28 @@ mod tests { value: None, }, }, -}"); - +}") + } else { + // untested without extension + }; } - #[test] - fn test_lookup_with_corrupt_data_returns_decoder_error() { + all_layout!(test_lookup_with_corrupt_data_returns_decoder_error, test_lookup_with_corrupt_data_returns_decoder_error_internal); + fn test_lookup_with_corrupt_data_returns_decoder_error_internal() { - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBA").unwrap(); } - let t = RefTrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::::new(&memdb, &root).unwrap(); // query for an invalid data type to trigger an error let q = |x: &[u8]| x.len() < 64; - let lookup = RefLookup { db: t.db(), query: q, hash: root }; + let lookup = Lookup:: { db: t.db(), query: q, hash: root }; let query_result = lookup.look_up(NibbleSlice::new(b"A")); assert_eq!(query_result.unwrap().unwrap(), true); } From dfba7bceb087b4691f9d795fa5024d93b85f0e1f Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 May 2020 10:00:32 +0200 Subject: [PATCH 56/73] Move macro to test-support --- test-support/reference-trie/src/lib.rs | 15 +++++++++ trie-db/src/iter_build.rs | 20 +++--------- trie-db/src/iterator.rs | 29 +++++------------ trie-db/src/triedb.rs | 29 +++++------------ trie-db/src/triedbmut.rs | 43 +++++++++++++------------- 5 files changed, 56 insertions(+), 80 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 7512806e..23da7d38 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -55,6 +55,21 @@ pub type RefHasher = ordered_trie::OrderedTrieHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; +#[macro_export] +macro_rules! test_layouts { + ($test:ident, $test_internal:ident) => { + #[test] + fn $test() { + $test_internal::(); + $test_internal::(); + $test_internal::(); + $test_internal::(); + } + }; +} + + + /// Trie layout using extension nodes. pub struct ExtensionLayout; diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 1a69c6fb..e93af61f 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -696,21 +696,9 @@ impl ProcessEncodedNode<::Out> for TrieRootUnhashe mod test { use crate::DBValue; use memory_db::{MemoryDB, HashKey, PrefixedKey}; - use reference_trie::{RefHasher, TrieLayout}; + use reference_trie::{RefHasher, TrieLayout, test_layouts}; use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; - macro_rules! all_layout { - ($test:ident, $test_internal:ident) => { - #[test] - fn $test() { - $test_internal::(); - $test_internal::(); - $test_internal::(); - $test_internal::(); - } - }; - } - #[test] fn trie_root_empty () { compare_implementations(vec![]) @@ -821,7 +809,7 @@ mod test { (vec![1u8, 2u8, 3u8, 5u8, 3u8], vec![7u8;32]), ]); } - all_layout!(root_extension_bis, root_extension_bis_internal); + test_layouts!(root_extension_bis, root_extension_bis_internal); fn root_extension_bis_internal() { compare_root::(vec![ (vec![1u8, 2u8, 3u8, 3u8], vec![8u8;32]), @@ -913,7 +901,7 @@ mod test { (vec![0x02, 0x50], vec![0x3]), ]); } - all_layout!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); + test_layouts!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); fn fuzz_no_extension_insert_remove_1_internal() { let data = vec![ (false, vec![0], vec![251, 255]), @@ -923,7 +911,7 @@ mod test { ]; compare_insert_remove::(data); } - all_layout!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); + test_layouts!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); fn fuzz_no_extension_insert_remove_2_internal() { let data = vec![ (false, vec![0x00], vec![0xfd, 0xff]), diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index f1ceb7c4..b944a3a7 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -391,21 +391,8 @@ mod tests { use reference_trie::{ TrieDB, TrieDBMut, TrieError, TrieMut, TrieIterator, TrieDBNodeIterator, NibbleSlice, NibbleVec, - node::Node, TrieLayout, + node::Node, TrieLayout, test_layouts, }; - use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; - - macro_rules! all_layout { - ($test:ident, $test_internal:ident) => { - #[test] - fn $test() { - $test_internal::(); - $test_internal::(); - $test_internal::(); - $test_internal::(); - } - }; - } type MemoryDB = memory_db::MemoryDB<::Hash, memory_db::PrefixedKey<::Hash>, DBValue>; @@ -433,7 +420,7 @@ mod tests { v } - all_layout!(iterator_works, iterator_works_internal); + test_layouts!(iterator_works, iterator_works_internal); fn iterator_works_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -532,7 +519,7 @@ mod tests { assert!(iter.next().is_none()); } - all_layout!(iterator_over_empty_works, iterator_over_empty_works_internal); + test_layouts!(iterator_over_empty_works, iterator_over_empty_works_internal); fn iterator_over_empty_works_internal() { let (memdb, root) = build_trie_db::(&[]); let trie = TrieDB::::new(&memdb, &root).unwrap(); @@ -552,7 +539,7 @@ mod tests { assert!(iter.next().is_none()); } - all_layout!(seek_works, seek_works_internal); + test_layouts!(seek_works, seek_works_internal); fn seek_works_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -596,7 +583,7 @@ mod tests { assert!(iter.next().is_none()); } - all_layout!(seek_over_empty_works, seek_over_empty_works_internal); + test_layouts!(seek_over_empty_works, seek_over_empty_works_internal); fn seek_over_empty_works_internal() { let (memdb, root) = build_trie_db::(&[]); let trie = TrieDB::::new(&memdb, &root).unwrap(); @@ -618,7 +605,7 @@ mod tests { assert!(iter.next().is_none()); } - all_layout!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); + test_layouts!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); fn iterate_over_incomplete_db_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -695,7 +682,7 @@ mod tests { } } - all_layout!(prefix_works, prefix_works_internal); + test_layouts!(prefix_works, prefix_works_internal); fn prefix_works_internal() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -756,7 +743,7 @@ mod tests { assert!(iter.next().is_none()); } - all_layout!(prefix_over_empty_works, prefix_over_empty_works_internal); + test_layouts!(prefix_over_empty_works, prefix_over_empty_works_internal); fn prefix_over_empty_works_internal() { let (memdb, root) = build_trie_db::(&[]); let trie = TrieDB::::new(&memdb, &root).unwrap(); diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 51f2f28a..94d674ee 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -332,23 +332,10 @@ impl<'a, L: TrieLayout> Iterator for TrieDBIterator<'a, L> { mod tests { use memory_db::{MemoryDB, PrefixedKey}; use crate::DBValue; - use reference_trie::{TrieDB, TrieDBMut, Lookup, Trie, TrieMut, NibbleSlice, TrieLayout}; - use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; + use reference_trie::{TrieDB, TrieDBMut, Lookup, Trie, TrieMut, NibbleSlice, TrieLayout, test_layouts}; use hex_literal::hex; - macro_rules! all_layout { - ($test:ident, $test_internal:ident) => { - #[test] - fn $test() { - $test_internal::(); - $test_internal::(); - $test_internal::(); - $test_internal::(); - } - }; - } - - all_layout!(iterator_works, iterator_works_internal); + test_layouts!(iterator_works, iterator_works_internal); fn iterator_works_internal() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), @@ -376,7 +363,7 @@ mod tests { assert_eq!(pairs, iter_pairs); } - all_layout!(iterator_seek_works, iterator_seek_works_internal); + test_layouts!(iterator_seek_works, iterator_seek_works_internal); fn iterator_seek_works_internal() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), @@ -419,7 +406,7 @@ mod tests { ); } - all_layout!(iterator, iterator_internal); + test_layouts!(iterator, iterator_internal); fn iterator_internal() { let d = vec![ b"A".to_vec(), @@ -450,7 +437,7 @@ mod tests { assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::>()); } - all_layout!(iterator_seek, iterator_seek_internal); + test_layouts!(iterator_seek, iterator_seek_internal); fn iterator_seek_internal() { let d = vec![ b"A".to_vec(), @@ -496,7 +483,7 @@ mod tests { assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } - all_layout!(get_length_with_extension, get_length_with_extension_internal); + test_layouts!(get_length_with_extension, get_length_with_extension_internal); fn get_length_with_extension_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -512,7 +499,7 @@ mod tests { assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()).unwrap(), None); } - all_layout!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); + test_layouts!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); fn debug_output_supports_pretty_print_internal() { let d = vec![ b"A".to_vec(), @@ -589,7 +576,7 @@ mod tests { }; } - all_layout!(test_lookup_with_corrupt_data_returns_decoder_error, test_lookup_with_corrupt_data_returns_decoder_error_internal); + test_layouts!(test_lookup_with_corrupt_data_returns_decoder_error, test_lookup_with_corrupt_data_returns_decoder_error_internal); fn test_lookup_with_corrupt_data_returns_decoder_error_internal() { let mut memdb = MemoryDB::, DBValue>::default(); diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index ab7e4ae6..c591e4db 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1690,14 +1690,13 @@ mod tests { use crate::DBValue; use memory_db::{MemoryDB, PrefixedKey}; use hash_db::Hasher; - use reference_trie::{TrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, + use reference_trie::{TrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, test_layouts, ReferenceNodeCodec, ReferenceNodeCodecNoExt, reference_trie_root_iter_build as reference_trie_root, TrieLayout}; use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; - use crate::nibble::BackingByteVec; - macro_rules! all_layout { + macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { #[test] fn $test() { @@ -1791,7 +1790,7 @@ mod tests { } } - all_layout!(init, init_internal); + test_layouts!(init, init_internal); fn init_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1800,7 +1799,7 @@ mod tests { assert_eq!(*t.root(), hashed_null_node); } - all_layout!(insert_on_empty, insert_on_empty_internal); + test_layouts!(insert_on_empty, insert_on_empty_internal); fn insert_on_empty_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1812,7 +1811,7 @@ mod tests { ); } - all_layout!(remove_to_empty, remove_to_empty_internal); + test_layouts!(remove_to_empty, remove_to_empty_internal); fn remove_to_empty_internal() { let big_value = b"00000000000000000000000000000000"; @@ -1826,7 +1825,7 @@ mod tests { t.remove(&[0x01]).unwrap(); } - all_layout!(remove_to_empty_no_extension, remove_to_empty_no_extension_internal); + test_layouts!(remove_to_empty_no_extension, remove_to_empty_no_extension_internal); fn remove_to_empty_no_extension_internal() { let big_value = b"00000000000000000000000000000000"; let big_value2 = b"00000000000000000000000000000002"; @@ -1849,7 +1848,7 @@ mod tests { ])); } - all_layout!(insert_replace_root, insert_replace_root_internal); + test_layouts!(insert_replace_root, insert_replace_root_internal); fn insert_replace_root_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1862,7 +1861,7 @@ mod tests { ); } - all_layout!(insert_make_branch_root, insert_make_branch_root_internal); + test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); fn insert_make_branch_root_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1875,7 +1874,7 @@ mod tests { ])); } - all_layout!(insert_into_branch_root, insert_into_branch_root_internal); + test_layouts!(insert_into_branch_root, insert_into_branch_root_internal); fn insert_into_branch_root_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1890,7 +1889,7 @@ mod tests { ])); } - all_layout!(insert_value_into_branch_root, insert_value_into_branch_root_internal); + test_layouts!(insert_value_into_branch_root, insert_value_into_branch_root_internal); fn insert_value_into_branch_root_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1903,7 +1902,7 @@ mod tests { ])); } - all_layout!(insert_split_leaf, insert_split_leaf_internal); + test_layouts!(insert_split_leaf, insert_split_leaf_internal); fn insert_split_leaf_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1916,7 +1915,7 @@ mod tests { ])); } - all_layout!(insert_split_extenstion, insert_split_extenstion_internal); + test_layouts!(insert_split_extenstion, insert_split_extenstion_internal); fn insert_split_extenstion_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1931,7 +1930,7 @@ mod tests { ])); } - all_layout!(insert_big_value, insert_big_value_internal); + test_layouts!(insert_big_value, insert_big_value_internal); fn insert_big_value_internal() { let big_value0 = b"00000000000000000000000000000000"; let big_value1 = b"11111111111111111111111111111111"; @@ -1947,7 +1946,7 @@ mod tests { ])); } - all_layout!(insert_duplicate_value, insert_duplicate_value_internal); + test_layouts!(insert_duplicate_value, insert_duplicate_value_internal); fn insert_duplicate_value_internal() { let big_value = b"00000000000000000000000000000000"; @@ -1962,7 +1961,7 @@ mod tests { ])); } - all_layout!(test_at_empty, test_at_empty_internal); + test_layouts!(test_at_empty, test_at_empty_internal); fn test_at_empty_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1970,7 +1969,7 @@ mod tests { assert_eq!(t.get(&[0x5]).unwrap(), None); } - all_layout!(test_at_one, test_at_one_internal); + test_layouts!(test_at_one, test_at_one_internal); fn test_at_one_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -1981,7 +1980,7 @@ mod tests { assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); } - all_layout!(test_at_three, test_at_three_internal); + test_layouts!(test_at_three, test_at_three_internal); fn test_at_three_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -2000,7 +1999,7 @@ mod tests { assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); } - all_layout!(stress, stress_internal); + test_layouts!(stress, stress_internal); fn stress_internal() { let mut seed = Default::default(); for _ in 0..50 { @@ -2038,7 +2037,7 @@ mod tests { } } - all_layout!(test_trie_existing, test_trie_existing_internal); + test_layouts!(test_trie_existing, test_trie_existing_internal); fn test_trie_existing_internal() { let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -2052,7 +2051,7 @@ mod tests { } } - all_layout!(insert_empty, insert_empty_internal); + test_layouts!(insert_empty, insert_empty_internal); fn insert_empty_internal() { let mut seed = Default::default(); let x = StandardMap { @@ -2081,7 +2080,7 @@ mod tests { assert_eq!(*t.root(), hashed_null_node); } - all_layout!(return_old_values, return_old_values_internal); + test_layouts!(return_old_values, return_old_values_internal); fn return_old_values_internal() { let mut seed = Default::default(); let x = StandardMap { From f97435339a946f3a7f86180a56ce5c65f36cda41 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 May 2020 10:03:32 +0200 Subject: [PATCH 57/73] remove dup macro --- trie-db/src/triedbmut.rs | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index c591e4db..37320b9c 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1692,23 +1692,8 @@ mod tests { use hash_db::Hasher; use reference_trie::{TrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, test_layouts, ReferenceNodeCodec, ReferenceNodeCodecNoExt, reference_trie_root_iter_build as reference_trie_root, TrieLayout}; - use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; use crate::nibble::BackingByteVec; - - macro_rules! test_layouts { - ($test:ident, $test_internal:ident) => { - #[test] - fn $test() { - $test_internal::(); - $test_internal::(); - $test_internal::(); - $test_internal::(); - } - }; - } - - fn populate_trie<'db, T: TrieLayout>( db: &'db mut dyn HashDBHybridDyn, root: &'db mut ::Out, @@ -1738,14 +1723,7 @@ mod tests { } } - #[test] - fn playpen() { - env_logger::init(); - playpen_internal::(); - playpen_internal::(); - playpen_internal::(); - playpen_internal::(); - } + test_layouts!(playpen, playpen_internal); fn playpen_internal() { let mut seed = Default::default(); for test_i in 0..10 { From ecba5654ae1ae71c112fd7efc5546d819513ff15 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 May 2020 11:37:39 +0200 Subject: [PATCH 58/73] parameterize compact proof tests --- test-support/reference-trie/src/lib.rs | 3 - trie-db/src/proof/mod.rs | 152 ++++++++++++------------- trie-db/src/recorder.rs | 23 +++- trie-db/src/trie_codec.rs | 52 ++------- trie-db/src/triedbmut.rs | 9 +- 5 files changed, 106 insertions(+), 133 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index d37a64a1..cfc32fd2 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -68,12 +68,9 @@ macro_rules! test_layouts { }; } - - /// Trie layout using extension nodes. pub struct ExtensionLayout; - impl TrieLayout for ExtensionLayout { const USE_EXTENSION: bool = true; const HYBRID_HASH: bool = false; diff --git a/trie-db/src/proof/mod.rs b/trie-db/src/proof/mod.rs index 307704ee..e4dcdffb 100644 --- a/trie-db/src/proof/mod.rs +++ b/trie-db/src/proof/mod.rs @@ -40,7 +40,7 @@ mod verify; mod tests { use hash_db::Hasher; use reference_trie::{ - ExtensionLayout, NoExtensionLayout, + NoExtensionLayout, test_layouts, proof::{generate_proof, verify_proof, VerifyError}, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut, }; @@ -95,28 +95,9 @@ mod tests { (root, proof, items) } - #[test] - fn trie_proof_works_with_ext() { - let (root, proof, items) = test_generate_proof::( - test_entries(), - vec![ - b"do", - b"dog", - b"doge", - b"bravo", - b"alfabet", // None, not found under leaf node - b"d", // None, witness is extension node with omitted child - b"do\x10", // None, empty branch child - b"halp", // None, witness is extension node with non-omitted child - ], - ); - - verify_proof::(&root, &proof, items.iter()).unwrap(); - } - - #[test] - fn trie_proof_works_without_ext() { - let (root, proof, items) = test_generate_proof::( + test_layouts!(trie_proof_works, trie_proof_works_internal); + fn trie_proof_works_internal() { + let (root, proof, items) = test_generate_proof::( test_entries(), vec![ b"do", @@ -130,12 +111,12 @@ mod tests { ], ); - verify_proof::(&root, &proof, items.iter()).unwrap(); + verify_proof::(&root, &proof, items.iter()).unwrap(); } - #[test] - fn trie_proof_works_for_empty_trie() { - let (root, proof, items) = test_generate_proof::( + test_layouts!(trie_proof_works_for_empty_trie, trie_proof_works_for_empty_trie_internal); + fn trie_proof_works_for_empty_trie_internal() { + let (root, proof, items) = test_generate_proof::( vec![], vec![ b"alpha", @@ -144,12 +125,12 @@ mod tests { ], ); - verify_proof::(&root, &proof, items.iter()).unwrap(); + verify_proof::(&root, &proof, items.iter()).unwrap(); } - #[test] - fn test_verify_duplicate_keys() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_duplicate_keys, test_verify_duplicate_keys_internal); + fn test_verify_duplicate_keys_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"bravo"], ); @@ -158,15 +139,18 @@ mod tests { (b"bravo", Some(b"bravo")), (b"bravo", Some(b"bravo")), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::DuplicateKey(b"bravo".to_vec())) + assert!( + if let Err(VerifyError::DuplicateKey(key)) = verify_proof::(&root, &proof, items.iter()) { + key == b"bravo".to_vec() + } else { + false + } ); } - #[test] - fn test_verify_extraneous_node() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_extraneaous_node, test_verify_extraneaous_node_internal); + fn test_verify_extraneaous_node_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"bravo", b"do"], ); @@ -174,15 +158,15 @@ mod tests { let items = vec![ (b"bravo", Some(b"bravo")), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), + assert!(matches!( + verify_proof::(&root, &proof, items.iter()), Err(VerifyError::ExtraneousNode) - ); + )); } - #[test] - fn test_verify_extraneous_value() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_extraneaous_value, test_verify_extraneaous_value_internal); + fn test_verify_extraneaous_value_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"doge"], ); @@ -191,18 +175,18 @@ mod tests { (&b"do"[..], Some(&b"verb"[..])), (&b"doge"[..], Some(&[0; 32][..])), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::ExtraneousValue(b"do".to_vec())) + assert!( + if let Err(VerifyError::ExtraneousValue(val)) = verify_proof::(&root, &proof, items.iter()) { + val == b"do".to_vec() + } else { + false + } ); } - // #[test] TODO this does not work for complex hash, restore - // when two test case. - // Reason it does not work is that the hash not in root are - // skipped so there is no extranous hash, error could be better - // still, but produce in advance children + #[test] fn test_verify_extraneous_hash_reference() { + // This is not valid for hybrid let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"do"], @@ -218,9 +202,9 @@ mod tests { } } - #[test] - fn test_verify_invalid_child_reference() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_invalid_child_reference, test_verify_invalid_child_reference_internal); + fn test_verify_invalid_child_reference_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"bravo"], ); @@ -230,15 +214,15 @@ mod tests { let items = vec![ (b"bravo", Some([0; 32])), ]; - match verify_proof::(&root, &proof, items.iter()) { + match verify_proof::(&root, &proof, items.iter()) { Err(VerifyError::InvalidChildReference(_)) => {} result => panic!("expected VerifyError::InvalidChildReference, got {:?}", result), } } - #[test] - fn test_verify_value_mismatch_some_to_none() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_value_mismatch_some_to_none, test_verify_value_mismatch_some_to_none_internal); + fn test_verify_value_mismatch_some_to_none_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"horse"], ); @@ -247,15 +231,18 @@ mod tests { (&b"horse"[..], Some(&b"stallion"[..])), (&b"halp"[..], Some(&b"plz"[..])), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::ValueMismatch(b"halp".to_vec())) + assert!( + if let Err(VerifyError::ValueMismatch(val)) = verify_proof::(&root, &proof, items.iter()) { + val == b"halp".to_vec() + } else { + false + } ); } - #[test] - fn test_verify_value_mismatch_none_to_some() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_value_mismatch_none_to_some, test_verify_value_mismatch_none_to_some_internal); + fn test_verify_value_mismatch_none_to_some_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"alfa", b"bravo"], ); @@ -264,29 +251,32 @@ mod tests { (&b"alfa"[..], Some(&[0; 32][..])), (&b"bravo"[..], None), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::ValueMismatch(b"bravo".to_vec())) + assert!( + if let Err(VerifyError::ValueMismatch(val)) = verify_proof::(&root, &proof, items.iter()) { + val == b"bravo".to_vec() + } else { + false + } ); } - #[test] - fn test_verify_incomplete_proof() { - let (root, mut proof, items) = test_generate_proof::( + test_layouts!(test_verify_incomplete_proof, test_verify_incomplete_proof_internal); + fn test_verify_incomplete_proof_internal() { + let (root, mut proof, items) = test_generate_proof::( test_entries(), vec![b"alfa"], ); proof.pop(); - assert_eq!( - verify_proof::(&root, &proof, items.iter()), + assert!(matches!( + verify_proof::(&root, &proof, items.iter()), Err(VerifyError::IncompleteProof) - ); + )); } - #[test] - fn test_verify_root_mismatch() { - let (root, proof, _) = test_generate_proof::( + test_layouts!(test_verify_root_mismatch, test_verify_root_mismatch_internal); + fn test_verify_root_mismatch_internal() { + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"bravo"], ); @@ -294,21 +284,21 @@ mod tests { let items = vec![ (b"bravo", Some("incorrect")), ]; - match verify_proof::(&root, &proof, items.iter()) { + match verify_proof::(&root, &proof, items.iter()) { Err(VerifyError::RootMismatch(_)) => {} result => panic!("expected VerifyError::RootMismatch, got {:?}", result), } } - #[test] - fn test_verify_decode_error() { - let (root, mut proof, items) = test_generate_proof::( + test_layouts!(test_verify_decode_error, test_verify_decode_error_internal); + fn test_verify_decode_error_internal() { + let (root, mut proof, items) = test_generate_proof::( test_entries(), vec![b"bravo"], ); proof.insert(0, b"this is not a trie node".to_vec()); - match verify_proof::(&root, &proof, items.iter()) { + match verify_proof::(&root, &proof, items.iter()) { Err(VerifyError::DecodeError(_)) => {} result => panic!("expected VerifyError::DecodeError, got {:?}", result), } diff --git a/trie-db/src/recorder.rs b/trie-db/src/recorder.rs index 69246623..474a7960 100644 --- a/trie-db/src/recorder.rs +++ b/trie-db/src/recorder.rs @@ -79,7 +79,7 @@ impl Recorder { mod tests { use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; - use reference_trie::{RefTrieDB, RefTrieDBMut, Trie, TrieMut, Recorder, Record, RefHasher}; + use reference_trie::{Trie, TrieMut, Recorder, Record, RefHasher}; #[test] fn basic_recorder() { @@ -131,12 +131,25 @@ mod tests { }); } - // #[test] TODO put it back for reftriedb not complex + #[test] fn trie_record() { - let mut db = MemoryDB::, _>::default(); + type KeccakHasher = ordered_trie::OrderedTrieHasher; + struct Layout; + + + impl reference_trie::TrieLayout for Layout { + const USE_EXTENSION: bool = true; + const HYBRID_HASH: bool = false; + type Hash = KeccakHasher; + type Codec = reference_trie::ReferenceNodeCodec; + } + + impl reference_trie::TrieConfiguration for Layout { } + + let mut db = MemoryDB::, _>::default(); let mut root = Default::default(); { - let mut x = RefTrieDBMut::new(&mut db, &mut root); + let mut x = reference_trie::TrieDBMut::::new(&mut db, &mut root); x.insert(b"dog", b"cat").unwrap(); x.insert(b"lunch", b"time").unwrap(); @@ -148,7 +161,7 @@ mod tests { x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); } - let trie = RefTrieDB::new(&db, &root).unwrap(); + let trie = reference_trie::TrieDB::::new(&db, &root).unwrap(); let mut recorder = Recorder::new(); trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 12aec444..1a9e8288 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -649,9 +649,8 @@ mod tests { use crate::DBValue; use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use reference_trie::{ - ExtensionLayout, NoExtensionLayout, Trie, TrieMut, TrieDB, TrieError, TrieDBMut, TrieLayout, Recorder, - encode_compact, decode_compact, + encode_compact, decode_compact, test_layouts, }; type MemoryDB = memory_db::MemoryDB, DBValue>; @@ -723,9 +722,9 @@ mod tests { } } - #[test] - fn trie_compact_encoding_works_with_ext() { - let (root, mut encoded, items) = test_encode_compact::( + test_layouts!(trie_compact_encoding_works, trie_compact_encoding_works_internal); + fn trie_compact_encoding_works_internal() { + let (root, mut encoded, items) = test_encode_compact::( vec![ // "alfa" is at a hash-referenced leaf node. (b"alfa", &[0; 32]), @@ -753,45 +752,12 @@ mod tests { ); encoded.push(Vec::new()); // Add an extra item to ensure it is not read. - test_decode_compact::(&encoded, items, root, encoded.len() - 1); + test_decode_compact::(&encoded, items, root, encoded.len() - 1); } - #[test] - fn trie_compact_encoding_works_without_ext() { - let (root, mut encoded, items) = test_encode_compact::( - vec![ - // "alfa" is at a hash-referenced leaf node. - (b"alfa", &[0; 32]), - // "bravo" is at an inline leaf node. - (b"bravo", b"bravo"), - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at an inline leaf node. - (b"dog", b"puppy"), - // "doge" is at a hash-referenced leaf node. - (b"doge", &[0; 32]), - // extension node "o" (plus nibble) to next branch. - (b"horse", b"stallion"), - (b"house", b"building"), - ], - vec![ - b"do", - b"dog", - b"doge", - b"bravo", - b"d", // None, witness is a branch partial - b"do\x10", // None, witness is empty branch child - b"halp", // None, witness is branch partial - ], - ); - - encoded.push(Vec::new()); // Add an extra item to ensure it is not read. - test_decode_compact::(&encoded, items, root, encoded.len() - 1); - } - - #[test] - fn trie_decoding_fails_with_incomplete_database() { - let (_, encoded, _) = test_encode_compact::( + test_layouts!(trie_decoding_fails_with_incomplete_database, trie_decoding_fails_with_incomplete_database_internal); + fn trie_decoding_fails_with_incomplete_database_internal() { + let (_, encoded, _) = test_encode_compact::( vec![ (b"alfa", &[0; 32]), (b"bravo", b"bravo"), @@ -805,7 +771,7 @@ mod tests { // Reconstruct the partial DB from the compact encoding. let mut db = MemoryDB::default(); - match decode_compact::(&mut db, &encoded[..encoded.len() - 1]) { + match decode_compact::(&mut db, &encoded[..encoded.len() - 1]) { Err(err) => match *err { TrieError::IncompleteDatabase(_) => {} _ => panic!("got unexpected TrieError"), diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 1e9d628d..55eec3ce 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1724,7 +1724,14 @@ mod tests { } } - test_layouts!(playpen, playpen_internal); + #[test] + fn playpen() { + env_logger::init(); + playpen_internal::(); + playpen_internal::(); + playpen_internal::(); + playpen_internal::(); + } fn playpen_internal() { let mut seed = Default::default(); for test_i in 0..10 { From 94079987f96530a8211e96d08b6b24bb9b9a1b0d Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 May 2020 12:00:44 +0200 Subject: [PATCH 59/73] invalid proof parameterized --- trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs | 4 ++-- trie-db/fuzz/src/lib.rs | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs b/trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs index 565e7eb9..3112e353 100644 --- a/trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs +++ b/trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs @@ -5,5 +5,5 @@ use libfuzzer_sys::fuzz_target; use trie_db_fuzz::fuzz_that_verify_rejects_invalid_proofs; fuzz_target!(|data: &[u8]| { - fuzz_that_verify_rejects_invalid_proofs(data); -}); \ No newline at end of file + fuzz_that_verify_rejects_invalid_proofs::(data); +}); diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 36c0a183..f898f4a5 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -17,7 +17,6 @@ use hash_db::Hasher; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ calc_root, - ExtensionLayout, proof::{generate_proof, verify_proof}, reference_trie_root_iter_build as reference_trie_root, TrieDBMut, @@ -303,7 +302,7 @@ pub fn fuzz_that_trie_codec_proofs(input: &[u8]) { test_trie_codec_proof::(data, keys); } -pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) { +pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) { if input.len() < 4 { return; } @@ -331,7 +330,7 @@ pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) { return; } - let (root, proof, mut items) = test_generate_proof::(data, keys); + let (root, proof, mut items) = test_generate_proof::(data, keys); // Make one item at random incorrect. let items_idx = random_int % items.len(); @@ -340,7 +339,7 @@ pub fn fuzz_that_verify_rejects_invalid_proofs(input: &[u8]) { (_, value) if value.is_some() => *value = None, (_, value) => *value = Some(DBValue::new()), } - assert!(verify_proof::(&root, &proof, items.iter()).is_err()); + assert!(verify_proof::(&root, &proof, items.iter()).is_err()); } fn test_generate_proof( From 9e234f4cc482d2e2b96a1561e8053bdc246b7c66 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 May 2020 12:10:04 +0200 Subject: [PATCH 60/73] use non hybrid when needed --- test-support/reference-trie/src/lib.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index cfc32fd2..12cfde9a 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -174,12 +174,12 @@ pub fn reference_trie_root_iter_build(input: I) -> { let mut cb = trie_db::TrieRootHybrid::::default(); trie_visit::(data_sorted_unique(input), &mut cb); - cb.root.unwrap_or(Default::default()) + cb.root.unwrap_or_default() }, false => { let mut cb = trie_db::TrieRoot::::default(); trie_visit::(data_sorted_unique(input), &mut cb); - cb.root.unwrap_or(Default::default()) + cb.root.unwrap_or_default() }, } } @@ -1417,11 +1417,11 @@ pub fn calc_root( if T::HYBRID_HASH { let mut cb = TrieRootHybrid::::default(); trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + cb.root.unwrap_or_default() } else { let mut cb = TrieRoot::::default(); trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + cb.root.unwrap_or_default() } } @@ -1440,11 +1440,11 @@ pub fn calc_root_build( if T::HYBRID_HASH { let mut cb = TrieBuilderHybrid::new(hashdb); trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + cb.root.unwrap_or_default() } else { let mut cb = TrieBuilder::new(hashdb); trie_visit::(data.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) + cb.root.unwrap_or_default() } } @@ -1465,11 +1465,7 @@ pub fn compare_implementations_unordered(b_map.into_iter(), &mut cb); - cb.root.unwrap_or(Default::default()) - }; + let root_new = calc_root_build::(b_map.into_iter(), &mut hashdb); if root != root_new { { From 77072c3311878c2661e39525a91e6b82ee41d51a Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 May 2020 09:25:17 +0200 Subject: [PATCH 61/73] exe to get some number --- test-support/reference-trie/Cargo.toml | 8 + .../reference-trie/src/compare_proof_size.rs | 141 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 test-support/reference-trie/src/compare_proof_size.rs diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 282f8cb4..64b38375 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -17,6 +17,10 @@ ordered-trie = { path = "../../ordered-trie", default-features = false, version parity-scale-codec = { version = "1.0.3", features = ["derive"] } blake2-rfc = { version = "0.2.18", default-features = false } +# Deps for proof size build +trie-standardmap = { path = "../trie-standardmap", version = "0.15.2" } +memory-db = { path = "../../memory-db", version = "0.20.1" } + [dev-dependencies] trie-bench = { path = "../trie-bench", version = "0.21.1" } criterion = "0.2.8" @@ -25,6 +29,10 @@ criterion = "0.2.8" name = "bench" harness = false +[[bin]] +name = "proof_size" +path = "src/compare_proof_size.rs" + [features] default = ["std"] # no actual support for std, only to avoid a cargo issues diff --git a/test-support/reference-trie/src/compare_proof_size.rs b/test-support/reference-trie/src/compare_proof_size.rs new file mode 100644 index 00000000..789520a5 --- /dev/null +++ b/test-support/reference-trie/src/compare_proof_size.rs @@ -0,0 +1,141 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use trie_standardmap::{Alphabet, StandardMap, ValueMode}; +use hash_db::{ + HashDB, + EMPTY_PREFIX, +}; +use memory_db::{ + MemoryDB, + PrefixedKey, + HashKey, +}; +use reference_trie::{ + ExtensionLayoutHybrid, + ExtensionLayout, + TrieDBMut, + TrieMut, + TrieDB, + Trie, + TrieLayout, + Recorder, + encode_compact, +}; +use parity_scale_codec::{Decode, Input, Output, Encode, Compact, Error as CodecError}; + +type DBValue = Vec; + +fn main() { + let trie_size = [100, 1000, 10_000, 100_000]; + let number_key = [1, 10, 100, 1000, 10_000]; + for s in trie_size.iter() { + compare(*s, &number_key[..]) + } +} +fn compare(trie_size: u32, number_key: &[usize]) { + + let mut seed = Default::default(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: trie_size, + }.make_with(&mut seed); + let mut memdb = MemoryDB::<::Hash, PrefixedKey<_>, DBValue>::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMut::::new(&mut memdb, &mut root); + for i in 0..x.len() { + let key: &[u8]= &x[i].0; + let val: &[u8] = &x[i].1; + t.insert(key, val).unwrap(); + } + t.commit(); + } + let trie = >::new(&memdb, &root).unwrap(); + for n in number_key { + if *n < trie_size as usize { + // we test only existing key, missing key should have better overall compression(could try with pure random) + compare_inner(&trie, trie_size, &x[..*n], "standard") + } + } + + let mut memdb = MemoryDB::<::Hash, PrefixedKey<_>, DBValue>::default(); + let mut root = Default::default(); + { + let mut tcomp = TrieDBMut::::new(&mut memdb, &mut root); + for i in 0..x.len() { + let key: &[u8]= &x[i].0; + let val: &[u8] = &x[i].1; + tcomp.insert(key, val).unwrap(); + } + tcomp.commit(); + } + let trie = >::new(&memdb, &root).unwrap(); + for n in number_key { + if *n < trie_size as usize { + compare_inner(&trie, trie_size, &x[..*n], "complex") + } + } +} + + +fn compare_inner(trie: &TrieDB, trie_size: u32, keys: &[(Vec, Vec)], lay: &str) { + let mut recorder = Recorder::new(); + let items = { + let mut items = Vec::with_capacity(keys.len()); + for (key, _) in keys { + let value = trie.get_with(key.as_slice(), &mut recorder).unwrap(); + items.push((key, value)); + } + items + }; + + let mut std_proof_len = compact_size(items.len()); + let mut partial_db = , _>>::default(); + for record in recorder.drain() { + std_proof_len += compact_size(record.data.len()); + std_proof_len += record.data.len(); + partial_db.emplace(record.hash, EMPTY_PREFIX, record.data); + } + + let compact_trie = { + let partial_trie = >::new(&partial_db, &trie.root()).unwrap(); + encode_compact::(&partial_trie).unwrap() + }; + + let mut compact_proof_len = compact_size(compact_trie.len()); + for node in compact_trie.iter() { + compact_proof_len += compact_size(node.len()); + compact_proof_len += node.len(); + } + + let ratio: f64 = compact_proof_len as f64 / std_proof_len as f64; + println!( + "On {} {} size trie, {} proof, non compact: {} compact: {} ratio: {}", + lay, + trie_size, + keys.len(), + std_proof_len, + compact_proof_len, + ratio, + ); +} + +fn compact_size(len: usize) -> usize { + // TODO switch to range, this is slow + Compact::(len as u64).encode().len() +} From 6ae3e37b54c4b3daec3ece42621f8aeb40ef7c88 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 May 2020 12:13:59 +0200 Subject: [PATCH 62/73] bench related to compaction processing. --- hash-db/src/lib.rs | 2 +- .../reference-trie/src/compare_proof_size.rs | 35 ++-- test-support/reference-trie/src/lib.rs | 6 +- trie-db/benches/bench.rs | 168 +++++++++++++++++- 4 files changed, 188 insertions(+), 23 deletions(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index f53531d5..12fe76c1 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -284,7 +284,7 @@ pub trait HasherHybrid: BinaryHasher { fn hash_hybrid< I: Iterator::Out>>, >( - x: &[u8], + encoded_node: &[u8], nb_children: usize, children: I, buffer: &mut ::Buffer, diff --git a/test-support/reference-trie/src/compare_proof_size.rs b/test-support/reference-trie/src/compare_proof_size.rs index 789520a5..924a53b7 100644 --- a/test-support/reference-trie/src/compare_proof_size.rs +++ b/test-support/reference-trie/src/compare_proof_size.rs @@ -33,23 +33,25 @@ use reference_trie::{ Recorder, encode_compact, }; -use parity_scale_codec::{Decode, Input, Output, Encode, Compact, Error as CodecError}; +use parity_scale_codec::{Encode, Compact}; type DBValue = Vec; fn main() { let trie_size = [100, 1000, 10_000, 100_000]; let number_key = [1, 10, 100, 1000, 10_000]; + let size_value = 32; for s in trie_size.iter() { - compare(*s, &number_key[..]) + compare(*s, &number_key[..], size_value) } } -fn compare(trie_size: u32, number_key: &[usize]) { + +fn compare(trie_size: u32, number_key: &[usize], size_value: usize) { let mut seed = Default::default(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, + min_key: size_value, journal_key: 0, value_mode: ValueMode::Index, count: trie_size, @@ -76,18 +78,18 @@ fn compare(trie_size: u32, number_key: &[usize]) { let mut memdb = MemoryDB::<::Hash, PrefixedKey<_>, DBValue>::default(); let mut root = Default::default(); { - let mut tcomp = TrieDBMut::::new(&mut memdb, &mut root); + let mut thyb = TrieDBMut::::new(&mut memdb, &mut root); for i in 0..x.len() { let key: &[u8]= &x[i].0; let val: &[u8] = &x[i].1; - tcomp.insert(key, val).unwrap(); + thyb.insert(key, val).unwrap(); } - tcomp.commit(); + thyb.commit(); } let trie = >::new(&memdb, &root).unwrap(); for n in number_key { if *n < trie_size as usize { - compare_inner(&trie, trie_size, &x[..*n], "complex") + compare_inner(&trie, trie_size, &x[..*n], "hybrid") } } } @@ -95,23 +97,20 @@ fn compare(trie_size: u32, number_key: &[usize]) { fn compare_inner(trie: &TrieDB, trie_size: u32, keys: &[(Vec, Vec)], lay: &str) { let mut recorder = Recorder::new(); - let items = { - let mut items = Vec::with_capacity(keys.len()); - for (key, _) in keys { - let value = trie.get_with(key.as_slice(), &mut recorder).unwrap(); - items.push((key, value)); - } - items - }; + for (key, _) in keys { + let _ = trie.get_with(key.as_slice(), &mut recorder).unwrap(); + } - let mut std_proof_len = compact_size(items.len()); + let mut std_proof_len = 0; let mut partial_db = , _>>::default(); + let mut nb_elt = 0; for record in recorder.drain() { std_proof_len += compact_size(record.data.len()); std_proof_len += record.data.len(); partial_db.emplace(record.hash, EMPTY_PREFIX, record.data); + nb_elt += 1; } - + std_proof_len += compact_size(nb_elt); let compact_trie = { let partial_trie = >::new(&partial_db, &trie.root()).unwrap(); encode_compact::(&partial_trie).unwrap() diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 12cfde9a..cfac9bc6 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -51,9 +51,9 @@ pub mod node { } /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. -pub type RefHasher = ordered_trie::OrderedTrieHasher; -//pub type RefHasher = ordered_trie::OrderedTrieHasher; -//pub type RefHasher = ordered_trie::OrderedTrieHasher; +pub type RefHasher = ordered_trie::OrderedTrieHasher; +//pub type RefHasher = ordered_trie::OrderedTrieHasher; +//pub type RefHasher = ordered_trie::OrderedTrieHasher; #[macro_export] macro_rules! test_layouts { diff --git a/trie-db/benches/bench.rs b/trie-db/benches/bench.rs index 9f74d419..24d0989e 100644 --- a/trie-db/benches/bench.rs +++ b/trie-db/benches/bench.rs @@ -14,7 +14,7 @@ use criterion::{criterion_group, criterion_main, Bencher, black_box, Criterion}; -use trie_db::{NibbleSlice, proof::{generate_proof, verify_proof}, Trie}; +use trie_db::{NibbleSlice, proof::{generate_proof, verify_proof}, TrieLayout}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; use reference_trie::ExtensionLayout as Layout; @@ -36,6 +36,12 @@ criterion_group!(benches, trie_iteration, nibble_common_prefix, trie_proof_verification, + proof_build_dataset_standard, + proof_build_dataset_hybrid, + proof_build_compacting_standard, + proof_build_compacting_hybrid, + proof_build_change_standard, + proof_build_change_hybrid, ); criterion_main!(benches); @@ -463,6 +469,7 @@ fn trie_iteration(c: &mut Criterion) { fn trie_proof_verification(c: &mut Criterion) { use memory_db::HashKey; + use trie_db::Trie; let mut data = input_unsorted(29, 204800, 32); let mut keys = data[(data.len() / 3)..] @@ -497,3 +504,162 @@ fn trie_proof_verification(c: &mut Criterion) { }) ); } + +// bench build triedbmut as in proof size main from reference trie +// parameters are hadcoded. +fn proof_build_dataset(c: &mut Criterion, trie_size: u32, size_value: usize) { + use memory_db::PrefixedKey; + use trie_db::TrieMut; + + let mut seed = Default::default(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: size_value, + journal_key: 0, + value_mode: ValueMode::Index, + count: trie_size, + }.make_with(&mut seed); + let mut memdb = memory_db::MemoryDB::<::Hash, PrefixedKey<_>, Vec>::default(); + let mut root = Default::default(); + + c.bench_function("proof_build_dataset", move |b: &mut Bencher| + b.iter(|| { + let mut t = reference_trie::TrieDBMut::::new(&mut memdb, &mut root); + for i in 0..x.len() { + let key: &[u8]= &x[i].0; + let val: &[u8] = &x[i].1; + t.insert(key, val).unwrap(); + } + t.commit(); + }) + ); +} + +fn proof_build_dataset_standard(c: &mut Criterion) { + let trie_size = 1000; + let values_size = 32; + proof_build_dataset::(c, trie_size, values_size) +} + +fn proof_build_dataset_hybrid(c: &mut Criterion) { + let trie_size = 1000; + let values_size = 32; + proof_build_dataset::(c, trie_size, values_size) +} + +fn proof_build_compacting(c: &mut Criterion, trie_size: u32, size_value: usize, number_key: usize) { + use memory_db::{PrefixedKey, HashKey}; + use trie_db::TrieMut; + + let mut seed = Default::default(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: size_value, + journal_key: 0, + value_mode: ValueMode::Index, + count: trie_size, + }.make_with(&mut seed); + let mut memdb = memory_db::MemoryDB::<::Hash, PrefixedKey<_>, Vec>::default(); + let mut root = Default::default(); + { + let mut t = reference_trie::TrieDBMut::::new(&mut memdb, &mut root); + for i in 0..x.len() { + let key: &[u8]= &x[i].0; + let val: &[u8] = &x[i].1; + t.insert(key, val).unwrap(); + } + t.commit(); + } + + use reference_trie::{Trie, TrieDB}; + use hash_db::{EMPTY_PREFIX, HashDB}; + + let keys = &x[..number_key]; + let trie = >::new(&memdb, &root).unwrap(); + let mut recorder = reference_trie::Recorder::new(); + for (key, _) in keys { + let _ = trie.get_with(key.as_slice(), &mut recorder).unwrap(); + } + + let mut partial_db = , _>>::default(); + for record in recorder.drain() { + partial_db.emplace(record.hash, EMPTY_PREFIX, record.data); + } + let partial_trie = >::new(&partial_db, &trie.root()).unwrap(); + + c.bench_function("proof_build_compacting", move |b: &mut Bencher| + b.iter(|| { + reference_trie::encode_compact::(&partial_trie).unwrap() + }) + ); +} + +fn proof_build_compacting_standard(c: &mut Criterion) { + let trie_size = 1000; + let proof_keys = 10; + let values_size = 32; + proof_build_compacting::(c, trie_size, values_size, proof_keys) +} + +fn proof_build_compacting_hybrid(c: &mut Criterion) { + let trie_size = 1000; + let proof_keys = 10; + let values_size = 32; + proof_build_compacting::(c, trie_size, values_size, proof_keys) +} + +fn proof_build_change(c: &mut Criterion, trie_size: u32, size_value: usize, number_key: usize) { + use memory_db::PrefixedKey; + use trie_db::TrieMut; + + let mut seed = Default::default(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: size_value, + journal_key: 0, + value_mode: ValueMode::Index, + count: trie_size, + }.make_with(&mut seed); + let mut memdb = memory_db::MemoryDB::<::Hash, PrefixedKey<_>, Vec>::default(); + let mut root = Default::default(); + { + let mut t = reference_trie::TrieDBMut::::new(&mut memdb, &mut root); + for i in 0..x.len() { + let key: &[u8]= &x[i].0; + let val: &[u8] = &x[i].1; + t.insert(key, val).unwrap(); + } + t.commit(); + } + + let keys = &x[..number_key]; + let value = vec![213u8; size_value]; + + c.bench_function("proof_build_change", move |b: &mut Bencher| + b.iter(|| { + let mut memdb = memdb.clone(); + let mut root = root.clone(); + let mut t = reference_trie::TrieDBMut::::from_existing(&mut memdb, &mut root).unwrap(); + for i in 0..keys.len() { + let key: &[u8]= &keys[i].0; + let val: &[u8] = &value[..]; + t.insert(key, val).unwrap(); + } + t.commit(); + }) + ); +} + +fn proof_build_change_standard(c: &mut Criterion) { + let trie_size = 1000; + let change_keys = 10; + let values_size = 32; + proof_build_change::(c, trie_size, values_size, change_keys) +} + +fn proof_build_change_hybrid(c: &mut Criterion) { + let trie_size = 1000; + let change_keys = 10; + let values_size = 32; + proof_build_change::(c, trie_size, values_size, change_keys) +} From a0a344ee2876c66b6ab206d2b5450bcb3d1741bb Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Jun 2020 11:48:01 +0200 Subject: [PATCH 63/73] restore deleted adapter --- trie-db/src/lib.rs | 2 +- trie-db/src/node_codec.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index daf18cea..3bc98413 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -71,7 +71,7 @@ pub use self::recorder::{Recorder, Record}; pub use self::lookup::Lookup; pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops}; pub use crate::node_codec::{NodeCodec, NodeCodecHybrid, Partial, HashDBHybridDyn, ChildProofHeader, - Bitmap, BITMAP_LENGTH, HashesPlan}; + Bitmap, BITMAP_LENGTH, HashesPlan, hybrid_hash_node_adapter}; pub use crate::iter_build::{trie_visit, ProcessEncodedNode, TrieRootUnhashedHybrid, TrieBuilder, TrieRoot, TrieRootUnhashed, TrieRootHybrid, TrieBuilderHybrid}; pub use crate::iterator::TrieDBNodeIterator; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index 33862d09..4399f85b 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -378,3 +378,33 @@ impl Iterator for HashesPlan { (size, Some(size)) } } + +/// Adapter standard implementation to use with `HashDBInsertComplex` function. +/// This can be use as a callback to insert encoded node into a hashdb when +/// using `insert_hybrid` method. +pub fn hybrid_hash_node_adapter, Hasher: HasherHybrid>( + encoded_node: &[u8] +) -> crate::rstd::result::Result, ()> { + Codec::need_hybrid_proof(encoded_node).map(|hybrid| + if let Some((node, common)) = hybrid { + match node { + NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { + let nb_children = children.iter().filter(|v| v.is_some()).count(); + let children = children.iter().map(|o_range| o_range.as_ref().map(|range| { + range.as_hash(encoded_node) + })); + let mut buf = ::InnerHasher::init_buffer(); + Some(Hasher::hash_hybrid( + common.header(encoded_node), + nb_children, + children, + &mut buf, + )) + }, + _ => unreachable!("hybrid only touch branch node"), + } + } else { + None + } + ) +} From 3109fe5acb7e87f83d1a1f2a61dea943203d63c0 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Jun 2020 18:25:11 +0200 Subject: [PATCH 64/73] std test --- hash-db/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 12fe76c1..cb9b21b8 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -80,6 +80,7 @@ pub trait BinaryHasher: Hasher { fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out; } +#[cfg(std)] /// Test function to use on any binary buffer implementation. pub fn test_binary_hasher() { let size = ::LENGTH * 2; From dedc9284a79d2158b9e06c4e4c837223e4a5b478 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Jun 2020 18:39:01 +0200 Subject: [PATCH 65/73] fix --- hash-db/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index cb9b21b8..0603b65b 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -80,7 +80,7 @@ pub trait BinaryHasher: Hasher { fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out; } -#[cfg(std)] +#[cfg(feature = "std")] /// Test function to use on any binary buffer implementation. pub fn test_binary_hasher() { let size = ::LENGTH * 2; From f6e5bc312f079603a9f1eef7d8454574129ee3ad Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 22 Jun 2020 15:58:57 +0200 Subject: [PATCH 66/73] Also check content from unpacked nodes in compare_proof_size --- .../reference-trie/src/compare_proof_size.rs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test-support/reference-trie/src/compare_proof_size.rs b/test-support/reference-trie/src/compare_proof_size.rs index 924a53b7..a030f205 100644 --- a/test-support/reference-trie/src/compare_proof_size.rs +++ b/test-support/reference-trie/src/compare_proof_size.rs @@ -32,6 +32,7 @@ use reference_trie::{ TrieLayout, Recorder, encode_compact, + decode_compact, }; use parity_scale_codec::{Encode, Compact}; @@ -42,11 +43,11 @@ fn main() { let number_key = [1, 10, 100, 1000, 10_000]; let size_value = 32; for s in trie_size.iter() { - compare(*s, &number_key[..], size_value) + compare(*s, &number_key[..], size_value, true) } } -fn compare(trie_size: u32, number_key: &[usize], size_value: usize) { +fn compare(trie_size: u32, number_key: &[usize], size_value: usize, check_proof: bool) { let mut seed = Default::default(); let x = StandardMap { @@ -71,7 +72,7 @@ fn compare(trie_size: u32, number_key: &[usize], size_value: usize) { for n in number_key { if *n < trie_size as usize { // we test only existing key, missing key should have better overall compression(could try with pure random) - compare_inner(&trie, trie_size, &x[..*n], "standard") + compare_inner(&trie, trie_size, &x[..*n], "standard", check_proof) } } @@ -89,13 +90,13 @@ fn compare(trie_size: u32, number_key: &[usize], size_value: usize) { let trie = >::new(&memdb, &root).unwrap(); for n in number_key { if *n < trie_size as usize { - compare_inner(&trie, trie_size, &x[..*n], "hybrid") + compare_inner(&trie, trie_size, &x[..*n], "hybrid", check_proof) } } } -fn compare_inner(trie: &TrieDB, trie_size: u32, keys: &[(Vec, Vec)], lay: &str) { +fn compare_inner(trie: &TrieDB, trie_size: u32, keys: &[(Vec, Vec)], lay: &str, check_proof: bool) { let mut recorder = Recorder::new(); for (key, _) in keys { let _ = trie.get_with(key.as_slice(), &mut recorder).unwrap(); @@ -116,6 +117,16 @@ fn compare_inner(trie: &TrieDB, trie_size: u32, keys: &[(Vec(&partial_trie).unwrap() }; + if check_proof { + let mut memdb = , _>>::default(); + let (root, nbs) = decode_compact::(&mut memdb, &compact_trie).unwrap(); + println!("nb node in proof {}", nbs); + let trie = >::new(&memdb, &root).unwrap(); + for (key, value) in keys { + let v = trie.get(key).unwrap(); + assert_eq!(v.as_ref(), Some(value)); + } + } let mut compact_proof_len = compact_size(compact_trie.len()); for node in compact_trie.iter() { compact_proof_len += compact_size(node.len()); From 08a1411482f24534225007878547002d3b5d7737 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jun 2020 09:58:43 +0200 Subject: [PATCH 67/73] change bound to allow 100% run --- test-support/reference-trie/src/compare_proof_size.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-support/reference-trie/src/compare_proof_size.rs b/test-support/reference-trie/src/compare_proof_size.rs index a030f205..f55655c1 100644 --- a/test-support/reference-trie/src/compare_proof_size.rs +++ b/test-support/reference-trie/src/compare_proof_size.rs @@ -70,7 +70,7 @@ fn compare(trie_size: u32, number_key: &[usize], size_value: usize, check_proof: } let trie = >::new(&memdb, &root).unwrap(); for n in number_key { - if *n < trie_size as usize { + if *n <= trie_size as usize { // we test only existing key, missing key should have better overall compression(could try with pure random) compare_inner(&trie, trie_size, &x[..*n], "standard", check_proof) } @@ -89,7 +89,7 @@ fn compare(trie_size: u32, number_key: &[usize], size_value: usize, check_proof: } let trie = >::new(&memdb, &root).unwrap(); for n in number_key { - if *n < trie_size as usize { + if *n <= trie_size as usize { compare_inner(&trie, trie_size, &x[..*n], "hybrid", check_proof) } } From a9a176401231c0780ebbb14d6de8b03a3a80963d Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 2 Mar 2021 16:28:03 +0100 Subject: [PATCH 68/73] fix merges and tests --- memory-db/src/lib.rs | 2 +- test-support/keccak-hasher/src/lib.rs | 2 +- test-support/reference-trie/Cargo.toml | 2 +- .../reference-trie/src/compare_proof_size.rs | 2 + test-support/reference-trie/src/lib.rs | 63 ++++++++++++++++--- trie-db/Cargo.toml | 1 + trie-db/fuzz/Cargo.toml | 2 +- trie-db/src/lib.rs | 2 +- trie-db/src/trie_codec.rs | 2 +- trie-db/test/Cargo.toml | 1 + trie-db/test/src/iter_build.rs | 50 ++++++--------- trie-db/test/src/iterator.rs | 9 +-- trie-db/test/src/proof.rs | 52 +++++++++------ trie-db/test/src/recorder.rs | 10 +-- trie-db/test/src/trie_codec.rs | 4 +- trie-db/test/src/triedb.rs | 10 ++- trie-db/test/src/triedbmut.rs | 26 ++++---- 17 files changed, 144 insertions(+), 96 deletions(-) diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index f61c70db..ad78f891 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -24,7 +24,7 @@ pub use malloc_size_of::*; use hash_db::{HashDB, HashDBRef, PlainDB, PlainDBRef, Hasher as KeyHasher, AsHashDB, AsPlainDB, Prefix, HashDBHybrid, HasherHybrid, BinaryHasher}; -use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; +use parity_util_mem::{MallocSizeOf, MallocSizeOfOps, MallocShallowSizeOf}; #[cfg(feature = "std")] use std::{ collections::hash_map::Entry, diff --git a/test-support/keccak-hasher/src/lib.rs b/test-support/keccak-hasher/src/lib.rs index 40f4acb1..ce066c1c 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/test-support/keccak-hasher/src/lib.rs @@ -45,7 +45,7 @@ impl BinaryHasher for KeccakHasher { type Buffer = Keccak; fn init_buffer() -> Self::Buffer { - Keccak::new_keccak256() + Keccak::v256() } fn reset_buffer(buff: &mut Self::Buffer) { diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 73d6a3e8..c8a956f0 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -19,7 +19,7 @@ parity-scale-codec = { version = "2.0.0", features = ["derive"] } # Deps for proof size build trie-standardmap = { path = "../trie-standardmap", version = "0.15.2" } -memory-db = { path = "../../memory-db", version = "0.20.1" } +memory-db = { path = "../../memory-db", version = "0.26.0" } [dev-dependencies] trie-bench = { path = "../trie-bench", version = "0.27.0" } diff --git a/test-support/reference-trie/src/compare_proof_size.rs b/test-support/reference-trie/src/compare_proof_size.rs index f55655c1..c2fe7f00 100644 --- a/test-support/reference-trie/src/compare_proof_size.rs +++ b/test-support/reference-trie/src/compare_proof_size.rs @@ -25,6 +25,8 @@ use memory_db::{ use reference_trie::{ ExtensionLayoutHybrid, ExtensionLayout, +}; +use trie_db::{ TrieDBMut, TrieMut, TrieDB, diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index ce1c82c9..bfa687c7 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -36,12 +36,16 @@ use trie_db::{ ChildProofHeader, HashesPlan, binary_additional_hashes, + TrieDB, + TrieDBMut, + Bitmap, BITMAP_LENGTH, }; use std::borrow::Borrow; use trie_db::{ +// decode_compact, encode_compact, HashDBHybridDyn, nibble_ops, NodeCodec, NodeCodecHybrid, - Trie, TrieConfiguration, + Trie, TrieConfiguration, HashDBHybrid, TrieLayout, TrieMut, }; pub use trie_root::TrieStream; @@ -52,6 +56,7 @@ pub mod node { /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. pub type RefHasher = ordered_trie::OrderedTrieHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; +//use keccak_hasher::KeccakHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; #[macro_export] @@ -85,6 +90,7 @@ pub struct ExtensionLayoutHybrid; impl TrieLayout for ExtensionLayoutHybrid { const USE_EXTENSION: bool = true; + const ALLOW_EMPTY: bool = false; const HYBRID_HASH: bool = true; type Hash = RefHasher; type Codec = ReferenceNodeCodec; @@ -92,7 +98,6 @@ impl TrieLayout for ExtensionLayoutHybrid { impl TrieConfiguration for ExtensionLayoutHybrid { } - /// Trie layout without extension nodes, allowing /// generic hasher. pub struct GenericNoExtensionLayout(PhantomData); @@ -137,11 +142,10 @@ impl TrieConfiguration for GenericNoExtensionLayoutHybrid { /// Trie layout without extension nodes. pub type NoExtensionLayoutHybrid = GenericNoExtensionLayoutHybrid; - -pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; -pub type RefTrieDBMut<'a> = trie_db::TrieDBMut<'a, ExtensionLayout>; -pub type RefTrieDBMutNoExt<'a> = trie_db::TrieDBMut<'a, NoExtensionLayout>; -pub type RefTrieDBMutAllowEmpty<'a> = trie_db::TrieDBMut<'a, AllowEmptyLayout>; +pub type RefTrieDB<'a> = TrieDB<'a, ExtensionLayout>; +pub type RefTrieDBMut<'a> = TrieDBMut<'a, ExtensionLayout>; +pub type RefTrieDBMutNoExt<'a> = TrieDBMut<'a, NoExtensionLayout>; +pub type RefTrieDBMutAllowEmpty<'a> = TrieDBMut<'a, AllowEmptyLayout>; pub type RefFatDB<'a> = trie_db::FatDB<'a, ExtensionLayout>; pub type RefFatDBMut<'a> = trie_db::FatDBMut<'a, ExtensionLayout>; pub type RefSecTrieDB<'a> = trie_db::SecTrieDB<'a, ExtensionLayout>; @@ -167,6 +171,14 @@ pub fn reference_trie_root(input: I) -> (input: I) -> Vec where + I: IntoIterator, + A: AsRef<[u8]> + Ord + fmt::Debug, + B: AsRef<[u8]> + fmt::Debug, +{ + trie_root::unhashed_trie::(input) +} + fn data_sorted_unique(input: I) -> Vec<(A, B)> where I: IntoIterator, @@ -198,6 +210,14 @@ pub fn reference_trie_root_iter_build(input: I) -> (input: I) -> Vec where + I: IntoIterator, + A: AsRef<[u8]> + Ord + fmt::Debug, + B: AsRef<[u8]> + fmt::Debug, +{ + trie_root::unhashed_trie_no_extension::(input) +} + const EMPTY_TRIE: u8 = 0; const LEAF_NODE_OFFSET: u8 = 1; const EXTENSION_NODE_OFFSET: u8 = 128; @@ -1418,6 +1438,35 @@ pub fn compare_root>( assert_eq!(root, root_new); } +/// Compare trie builder and trie root unhashed implementations. +pub fn compare_unhashed( + data: Vec<(Vec, Vec)>, +) { + let root_new = { + let mut cb = trie_db::TrieRootUnhashed::::default(); + trie_visit::(data.clone().into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + }; + let root = reference_trie_root_unhashed(data); + + assert_eq!(root, root_new); +} + +/// Compare trie builder and trie root unhashed implementations. +/// This uses the variant without extension nodes. +pub fn compare_unhashed_no_extension( + data: Vec<(Vec, Vec)>, +) { + let root_new = { + let mut cb = trie_db::TrieRootUnhashed::::default(); + trie_visit::(data.clone().into_iter(), &mut cb); + cb.root.unwrap_or(Default::default()) + }; + let root = reference_trie_root_unhashed_no_extension(data); + + assert_eq!(root, root_new); +} + /// Trie builder root calculation utility. pub fn calc_root( data: I, diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index 64d8803e..d95ad9a7 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -12,6 +12,7 @@ log = "0.4" smallvec = "1.0.0" hash-db = { path = "../hash-db", default-features = false, version = "0.15.2"} hashbrown = { version = "0.9.1", default-features = false, features = ["ahash"] } +ordered-trie = { path = "../ordered-trie", default-features = false, version = "0.19.2"} rustc-hex = { version = "2.1.0", default-features = false, optional = true } [features] diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index c2edc99c..4f93f7cb 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -10,7 +10,7 @@ cargo-fuzz = true [dependencies] hash-db = { path = "../../hash-db", version = "0.15.2" } -memory-db = { path = "../../memory-db", version = "0.21.0" } +memory-db = { path = "../../memory-db", version = "0.26.0" } reference-trie = { path = "../../test-support/reference-trie", version = "0.21.0" } keccak-hasher = { path = "../../test-support/keccak-hasher", version = "0.15.2" } diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index f99f8933..25da53d3 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -38,7 +38,7 @@ mod rstd { } #[cfg(feature = "std")] -use self::rstd::fmt; +use self::rstd::{fmt, Error}; use hash_db::MaybeDebug; use self::rstd::{boxed::Box, vec::Vec}; diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 57ebfeb7..91eb21e1 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -511,7 +511,7 @@ pub fn decode_compact_from_iter<'a, L, DB, T, I>(db: &mut DB, encoded: I) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDB, + DB: HashDBHybrid, I: IntoIterator, { // The stack of nodes through a path in the trie. Each entry is a child node of the preceding diff --git a/trie-db/test/Cargo.toml b/trie-db/test/Cargo.toml index 00301870..0b05869b 100644 --- a/trie-db/test/Cargo.toml +++ b/trie-db/test/Cargo.toml @@ -25,3 +25,4 @@ hex-literal = "0.3" criterion = "0.3" env_logger = "0.8" log = "0.4" +ordered-trie = { path = "../../ordered-trie", default-features = false, version = "0.19.2"} diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index 70d2a5c0..be802bc7 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -12,21 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use trie_db::DBValue; +use trie_db::{DBValue, TrieLayout}; use memory_db::{MemoryDB, HashKey, PrefixedKey}; -use reference_trie::RefHasher; - -macro_rules! all_layout { - ($test:ident, $test_internal:ident) => { - #[test] - fn $test() { - $test_internal::(); - $test_internal::(); - $test_internal::(); - $test_internal::(); - } - }; -} +use reference_trie::{RefHasher, test_layouts, ExtensionLayoutHybrid, + NoExtensionLayout, ExtensionLayout, NoExtensionLayoutHybrid}; #[test] fn trie_root_empty () { @@ -49,8 +38,7 @@ fn root_extension_one () { } fn test_iter(data: Vec<(Vec, Vec)>) { - use reference_trie::{TrieMut, Trie, TrieDBMut, TrieDB}; - use trie_db::{TrieMut, Trie}; + use trie_db::{TrieMut, Trie, TrieDBMut, TrieDB}; let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); @@ -109,7 +97,8 @@ fn compare_implementations_h_internal(data: Vec<(Vec, Vec fn compare_implementations_no_extension_unordered(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::<_, HashKey<_>, _>::default(); let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations_no_extension_unordered(data, memdb, hashdb); + reference_trie::compare_implementations_unordered::(data.clone(), memdb.clone(), hashdb.clone()); + reference_trie::compare_implementations_unordered::(data, memdb, hashdb); } fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) { let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); @@ -145,9 +134,9 @@ fn trie_middle_node2 () { (vec![1u8, 2u8, 3u8, 5u8, 3u8], vec![7u8;32]), ]); } -#[test] -fn root_extension_bis () { - compare_root(vec![ +test_layouts!(root_extension_bis, root_extension_bis_internal); +fn root_extension_bis_internal() { + compare_root::(vec![ (vec![1u8, 2u8, 3u8, 3u8], vec![8u8;32]), (vec![1u8, 2u8, 3u8, 4u8], vec![7u8;32]), ]); @@ -251,14 +240,14 @@ fn fuzz_no_extension3 () { } #[test] fn fuzz_no_extension4 () { - compare_implementations_no_extension(vec![ + compare_implementations(vec![ (vec![0x01, 0x56], vec![0x1]), (vec![0x02, 0x42], vec![0x2]), (vec![0x02, 0x50], vec![0x3]), ]); } -all_layout!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); +test_layouts!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); fn fuzz_no_extension_insert_remove_1_internal() { let data = vec![ (false, vec![0], vec![251, 255]), @@ -266,10 +255,10 @@ fn fuzz_no_extension_insert_remove_1_internal() { (false, vec![0, 1, 2], vec![255; 32]), (true, vec![0, 1], vec![0, 251]), ]; - compare_no_extension_insert_remove::(data); + compare_insert_remove::(data); } -all_layout!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); +test_layouts!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); fn fuzz_no_extension_insert_remove_2_internal() { let data = vec![ (false, vec![0x00], vec![0xfd, 0xff]), @@ -277,7 +266,7 @@ fn fuzz_no_extension_insert_remove_2_internal() { (false, vec![0x11, 0x10], vec![0;32]), (true, vec![0x10, 0x00], vec![]) ]; - compare_no_extension_insert_remove::(data); + compare_insert_remove::(data); } #[test] fn two_bytes_nibble_length () { @@ -291,16 +280,17 @@ fn two_bytes_nibble_length () { #[test] #[should_panic] fn too_big_nibble_length_old () { - compare_implementations_prefixed_internal::( + compare_implementations_prefixed_internal::( vec![(vec![01u8;64], vec![0;32])], ); } #[test] fn too_big_nibble_length_new () { - // this is valid for no_ext code only, - // the other one got maximal length in encoding. - compare_implementations_prefixed_internal::(data.clone()); - compare_implementations_prefixed_internal::(data.clone()); + let data = vec![ + (vec![01u8;((u16::max_value() as usize + 1) / 2) + 1], vec![0;32]), + ]; + compare_implementations_prefixed_internal::(data.clone()); + compare_implementations_prefixed_internal::(data.clone()); } #[test] fn polka_re_test () { diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index bce7d102..6e66040c 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -15,14 +15,11 @@ use trie_db::{ DBValue, TrieError, TrieMut, TrieIterator, TrieDBNodeIterator, NibbleSlice, NibbleVec, - node::Node, + node::Node, TrieDB, TrieLayout, }; use hex_literal::hex; use hash_db::{HashDB, Hasher}; -use reference_trie::{ - RefTrieDB, RefTrieDBMut, RefHasher, -}; -use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt, test_layouts}; +use reference_trie::test_layouts; type MemoryDB = memory_db::MemoryDB<::Hash, memory_db::PrefixedKey<::Hash>, DBValue>; @@ -33,7 +30,7 @@ fn build_trie_db(pairs: &[(Vec, Vec)]) let mut memdb = MemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in pairs.iter() { t.insert(x, y).unwrap(); } diff --git a/trie-db/test/src/proof.rs b/trie-db/test/src/proof.rs index 0b9b6e45..41aba20a 100644 --- a/trie-db/test/src/proof.rs +++ b/trie-db/test/src/proof.rs @@ -14,7 +14,7 @@ use hash_db::Hasher; use reference_trie::{ - ExtensionLayout, NoExtensionLayout, + NoExtensionLayout, test_layouts, }; use trie_db::{ @@ -86,7 +86,7 @@ fn trie_proof_works_internal() { ], ); - verify_proof::(&root, &proof, items.iter()).unwrap(); + verify_proof::(&root, &proof, items.iter()).unwrap(); } test_layouts!(trie_proof_works_for_empty_trie, trie_proof_works_for_empty_trie_internal); @@ -114,9 +114,12 @@ fn test_verify_duplicate_keys_internal() { (b"bravo", Some(b"bravo")), (b"bravo", Some(b"bravo")), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::DuplicateKey(b"bravo".to_vec())) + assert!( + if let Err(VerifyError::DuplicateKey(key)) = verify_proof::(&root, &proof, items.iter()) { + key == b"bravo".to_vec() + } else { + false + } ); } @@ -130,10 +133,10 @@ fn test_verify_extraneaous_node_internal() { let items = vec![ (b"bravo", Some(b"bravo")), ]; - assert_eq!( + assert!(matches!( verify_proof::(&root, &proof, items.iter()), Err(VerifyError::ExtraneousNode) - ); + )); } test_layouts!(test_verify_extraneaous_value, test_verify_extraneaous_value_internal); @@ -147,9 +150,12 @@ fn test_verify_extraneaous_value_internal() { (&b"do"[..], Some(&b"verb"[..])), (&b"doge"[..], Some(&[0; 32][..])), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::ExtraneousValue(b"do".to_vec())) + assert!( + if let Err(VerifyError::ExtraneousValue(val)) = verify_proof::(&root, &proof, items.iter()) { + val == b"do".to_vec() + } else { + false + } ); } @@ -182,7 +188,7 @@ fn test_verify_invalid_child_reference_internal() { let items = vec![ (b"bravo", Some([0; 32])), ]; - match verify_proof::(&root, &proof, items.iter()) { + match verify_proof::(&root, &proof, items.iter()) { Err(VerifyError::InvalidChildReference(_)) => {} result => panic!("expected VerifyError::InvalidChildReference, got {:?}", result), } @@ -199,15 +205,18 @@ fn test_verify_value_mismatch_some_to_none_internal() { (&b"horse"[..], Some(&b"stallion"[..])), (&b"halp"[..], Some(&b"plz"[..])), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::ValueMismatch(b"halp".to_vec())) + assert!( + if let Err(VerifyError::ValueMismatch(val)) = verify_proof::(&root, &proof, items.iter()) { + val == b"halp".to_vec() + } else { + false + } ); } test_layouts!(test_verify_value_mismatch_none_to_some, test_verify_value_mismatch_none_to_some_internal); fn test_verify_value_mismatch_none_to_some_internal() { - let (root, proof, _) = test_generate_proof::( + let (root, proof, _) = test_generate_proof::( test_entries(), vec![b"alfa", b"bravo"], ); @@ -216,9 +225,12 @@ fn test_verify_value_mismatch_none_to_some_internal() { (&b"alfa"[..], Some(&[0; 32][..])), (&b"bravo"[..], None), ]; - assert_eq!( - verify_proof::(&root, &proof, items.iter()), - Err(VerifyError::ValueMismatch(b"bravo".to_vec())) + assert!( + if let Err(VerifyError::ValueMismatch(val)) = verify_proof::(&root, &proof, items.iter()) { + val == b"bravo".to_vec() + } else { + false + } ); } @@ -230,10 +242,10 @@ fn test_verify_incomplete_proof_internal() { ); proof.pop(); - assert_eq!( + assert!(matches!( verify_proof::(&root, &proof, items.iter()), Err(VerifyError::IncompleteProof) - ); + )); } test_layouts!(test_verify_root_mismatch, test_verify_root_mismatch_internal); diff --git a/trie-db/test/src/recorder.rs b/trie-db/test/src/recorder.rs index f7199059..7000992b 100644 --- a/trie-db/test/src/recorder.rs +++ b/trie-db/test/src/recorder.rs @@ -16,7 +16,7 @@ use memory_db::{MemoryDB, HashKey}; use hash_db::Hasher; -use reference_trie::{RefTrieDB, RefTrieDBMut, RefHasher}; +use reference_trie::RefHasher; use trie_db::{Trie, TrieMut, Recorder, Record}; #[test] @@ -74,19 +74,19 @@ fn trie_record() { type KeccakHasher = ordered_trie::OrderedTrieHasher; struct Layout; - impl reference_trie::TrieLayout for Layout { + impl trie_db::TrieLayout for Layout { const USE_EXTENSION: bool = true; const HYBRID_HASH: bool = false; type Hash = KeccakHasher; type Codec = reference_trie::ReferenceNodeCodec; } - impl reference_trie::TrieConfiguration for Layout { } + impl trie_db::TrieConfiguration for Layout { } let mut db = MemoryDB::, _>::default(); let mut root = Default::default(); { - let mut x = reference_trie::TrieDBMut::::new(&mut db, &mut root); + let mut x = trie_db::TrieDBMut::::new(&mut db, &mut root); x.insert(b"dog", b"cat").unwrap(); x.insert(b"lunch", b"time").unwrap(); @@ -98,7 +98,7 @@ fn trie_record() { x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); } - let trie = RefTrieDB::new(&db, &root).unwrap(); + let trie = trie_db::TrieDB::::new(&db, &root).unwrap(); let mut recorder = Recorder::new(); trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index b8e1e446..2394b3fa 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -18,9 +18,7 @@ use trie_db::{ Trie, TrieMut, TrieDB, TrieError, TrieDBMut, TrieLayout, Recorder, }; use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use reference_trie::{ - ExtensionLayout, NoExtensionLayout, -}; +use reference_trie::test_layouts; type MemoryDB = memory_db::MemoryDB, DBValue>; diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 9303ad74..12fb9908 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -13,11 +13,9 @@ // limitations under the License. use memory_db::{MemoryDB, PrefixedKey}; -use trie_db::{DBValue, Trie, TrieMut, NibbleSlice}; -use reference_trie::{RefTrieDB, RefTrieDBMut, RefLookup}; -use reference_trie::{RefTrieDBNoExt, RefTrieDBMutNoExt}; -use reference_trie::{TrieDB, TrieDBMut, Lookup, Trie, TrieMut, NibbleSlice, TrieLayout}; -use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; +use trie_db::{TrieDB, TrieDBMut, Lookup, Trie, TrieMut, NibbleSlice, TrieLayout, + DBValue}; +use reference_trie::test_layouts; use hex_literal::hex; test_layouts!(iterator_works, iterator_works_internal); @@ -66,7 +64,7 @@ fn iterator_seek_works_internal() { let t = TrieDB::::new(&memdb, &root).unwrap(); - let iter = trie.iter().unwrap(); + let mut iter = t.iter().unwrap(); assert_eq!( iter.next().unwrap().unwrap(), ( diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 5d7b380f..6d5fc248 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -15,22 +15,19 @@ use env_logger; use trie_standardmap::*; use log::debug; -use trie_db::{DBValue, TrieMut, NodeCodec,}; use memory_db::{MemoryDB, PrefixedKey}; -use hash_db::{Hasher, HashDB}; -use reference_trie::{RefTrieDBMutNoExt, RefTrieDBMutAllowEmpty, RefTrieDBMut, - ReferenceNodeCodec, reference_trie_root, reference_trie_root_no_extension}; -use reference_trie::{TrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, - ReferenceNodeCodec, ReferenceNodeCodecNoExt, reference_trie_root_iter_build as reference_trie_root, TrieLayout}; -use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, NoExtensionLayoutHybrid}; - -use crate::nibble::BackingByteVec; +use hash_db::Hasher; +use trie_db::{TrieDBMut, TrieMut, NodeCodec, HashDBHybridDyn, + TrieLayout, DBValue}; +use reference_trie::{ExtensionLayout, ExtensionLayoutHybrid, NoExtensionLayout, + NoExtensionLayoutHybrid, RefHasher, test_layouts, ReferenceNodeCodec, + ReferenceNodeCodecNoExt, reference_trie_root_iter_build as reference_trie_root}; fn populate_trie<'db, T: TrieLayout>( db: &'db mut dyn HashDBHybridDyn, root: &'db mut ::Out, v: &[(Vec, Vec)] -) -> RefTrieDBMut<'db> { +) -> TrieDBMut<'db, T> { let mut t = TrieDBMut::::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; @@ -187,7 +184,7 @@ test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); fn insert_make_branch_root_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); assert_eq!(*t.root(), reference_trie_root::(vec![ @@ -430,8 +427,11 @@ fn return_old_values_internal() { fn insert_empty_allowed() { let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = RefTrieDBMutAllowEmpty::new(&mut db, &mut root); + let mut t = reference_trie::RefTrieDBMutAllowEmpty::new(&mut db, &mut root); t.insert(b"test", &[]).unwrap(); - assert_eq!(*t.root(), reference_trie_root(vec![(b"test".to_vec(), Vec::new())])); + + assert_eq!(*t.root(), reference_trie_root::( + vec![(b"test".to_vec(), Vec::new())], + )); assert_eq!(t.get(b"test").unwrap(), Some(Vec::new())); } From 86e1dcd62977f7a5e317e585d2dcea7c78106361 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 10 Mar 2021 09:34:05 +0100 Subject: [PATCH 69/73] comment --- test-support/reference-trie/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index bfa687c7..846e69ec 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -59,6 +59,7 @@ pub type RefHasher = ordered_trie::OrderedTrieHasher; +/// Apply a test method on every test layouts. #[macro_export] macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { From aa537451af842f1046ce670a7a3b0721096d86f6 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 10 Mar 2021 10:00:18 +0100 Subject: [PATCH 70/73] fix bench check --- trie-db/test/benches/bench.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trie-db/test/benches/bench.rs b/trie-db/test/benches/bench.rs index 24d0989e..b8c85275 100644 --- a/trie-db/test/benches/bench.rs +++ b/trie-db/test/benches/bench.rs @@ -373,7 +373,7 @@ fn trie_mut_a(c: &mut Criterion) { let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = reference_trie::TrieDBMut::::new(&mut mdb, &mut root); + let mut trie = trie_db::TrieDBMut::::new(&mut mdb, &mut root); for (key, value) in datac { trie.insert(&key, &value) .expect("changes trie: insertion to trie is not allowed to fail within runtime"); @@ -397,7 +397,7 @@ fn trie_mut_b(c: &mut Criterion) { let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = reference_trie::TrieDBMut::::new(&mut mdb, &mut root); + let mut trie = trie_db::TrieDBMut::::new(&mut mdb, &mut root); for (key, value) in datac { trie.insert(&key, &value) .expect("changes trie: insertion to trie is not allowed to fail within runtime"); @@ -460,7 +460,7 @@ fn trie_iteration(c: &mut Criterion) { c.bench_function("trie_iteration", move |b: &mut Bencher| b.iter(|| { - let trie = reference_trie::TrieDB::::new(&mdb, &root).unwrap(); + let trie = trie_db::TrieDB::::new(&mdb, &root).unwrap(); let mut iter = trie_db::TrieDBNodeIterator::new(&trie).unwrap(); assert!(iter.all(|result| result.is_ok())); }) @@ -485,7 +485,7 @@ fn trie_proof_verification(c: &mut Criterion) { let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); let root = reference_trie::calc_root_build::(data, &mut mdb); - let trie = reference_trie::TrieDB::::new(&mdb, &root).unwrap(); + let trie = trie_db::TrieDB::::new(&mdb, &root).unwrap(); let proof = generate_proof(&trie, keys.iter()).unwrap(); let items = keys.into_iter() .map(|key| { From 6f20f374034b93fce16b7d397919077205fdf460 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 10 Mar 2021 10:30:46 +0100 Subject: [PATCH 71/73] fix bench --- trie-db/test/benches/bench.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/trie-db/test/benches/bench.rs b/trie-db/test/benches/bench.rs index b8c85275..0e03872a 100644 --- a/trie-db/test/benches/bench.rs +++ b/trie-db/test/benches/bench.rs @@ -524,7 +524,7 @@ fn proof_build_dataset(c: &mut Criterion, trie_size: u32, size_va c.bench_function("proof_build_dataset", move |b: &mut Bencher| b.iter(|| { - let mut t = reference_trie::TrieDBMut::::new(&mut memdb, &mut root); + let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); for i in 0..x.len() { let key: &[u8]= &x[i].0; let val: &[u8] = &x[i].1; @@ -562,7 +562,7 @@ fn proof_build_compacting(c: &mut Criterion, trie_size: u32, size let mut memdb = memory_db::MemoryDB::<::Hash, PrefixedKey<_>, Vec>::default(); let mut root = Default::default(); { - let mut t = reference_trie::TrieDBMut::::new(&mut memdb, &mut root); + let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); for i in 0..x.len() { let key: &[u8]= &x[i].0; let val: &[u8] = &x[i].1; @@ -571,12 +571,12 @@ fn proof_build_compacting(c: &mut Criterion, trie_size: u32, size t.commit(); } - use reference_trie::{Trie, TrieDB}; + use trie_db::{Trie, TrieDB}; use hash_db::{EMPTY_PREFIX, HashDB}; let keys = &x[..number_key]; let trie = >::new(&memdb, &root).unwrap(); - let mut recorder = reference_trie::Recorder::new(); + let mut recorder = trie_db::Recorder::new(); for (key, _) in keys { let _ = trie.get_with(key.as_slice(), &mut recorder).unwrap(); } @@ -589,7 +589,7 @@ fn proof_build_compacting(c: &mut Criterion, trie_size: u32, size c.bench_function("proof_build_compacting", move |b: &mut Bencher| b.iter(|| { - reference_trie::encode_compact::(&partial_trie).unwrap() + trie_db::encode_compact::(&partial_trie).unwrap() }) ); } @@ -623,7 +623,7 @@ fn proof_build_change(c: &mut Criterion, trie_size: u32, size_val let mut memdb = memory_db::MemoryDB::<::Hash, PrefixedKey<_>, Vec>::default(); let mut root = Default::default(); { - let mut t = reference_trie::TrieDBMut::::new(&mut memdb, &mut root); + let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); for i in 0..x.len() { let key: &[u8]= &x[i].0; let val: &[u8] = &x[i].1; @@ -639,7 +639,7 @@ fn proof_build_change(c: &mut Criterion, trie_size: u32, size_val b.iter(|| { let mut memdb = memdb.clone(); let mut root = root.clone(); - let mut t = reference_trie::TrieDBMut::::from_existing(&mut memdb, &mut root).unwrap(); + let mut t = trie_db::TrieDBMut::::from_existing(&mut memdb, &mut root).unwrap(); for i in 0..keys.len() { let key: &[u8]= &keys[i].0; let val: &[u8] = &value[..]; From 43a758ff3eb7ecc4249cc612a33cf989c62a3ef8 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 26 Mar 2021 10:29:18 +0100 Subject: [PATCH 72/73] blak3 bin hasher is broken --- test-support/reference-trie/Cargo.toml | 2 + test-support/reference-trie/src/lib.rs | 59 +++++++++++++++++++++++++- trie-db/src/triedbmut.rs | 4 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index c8a956f0..2094e200 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -14,6 +14,7 @@ keccak-hasher = { path = "../keccak-hasher", version = "0.15.3" } trie-db = { path = "../../trie-db", default-features = false, version = "0.22.0" } trie-root = { path = "../../trie-root", default-features = false, version = "0.16.0" } blake2-rfc = { version = "0.2.18", default-features = false } +blake3 = { version = "0.3.7", default-features = false } ordered-trie = { path = "../../ordered-trie", default-features = false, version = "0.19.2"} parity-scale-codec = { version = "2.0.0", features = ["derive"] } @@ -40,4 +41,5 @@ std = [ "trie-db/std", "trie-root/std", "blake2-rfc/std", + "blake3/std", ] diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 846e69ec..b39f7bb1 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -54,7 +54,8 @@ pub mod node { } /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. -pub type RefHasher = ordered_trie::OrderedTrieHasher; +//pub type RefHasher = ordered_trie::OrderedTrieHasher; +pub type RefHasher = ordered_trie::OrderedTrieHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; //use keccak_hasher::KeccakHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; @@ -1712,3 +1713,59 @@ pub mod blake2 { hash_db::test_binary_hasher::() } } + + +pub mod blake3 { + use hash_db::{Hasher, BinaryHasher}; + use hash256_std_hasher::Hash256StdHasher; + + /// Concrete implementation of Hasher using Blake2b 256-bit hashes + #[derive(Debug)] + pub struct Blake3Hasher; + + impl Hasher for Blake3Hasher { + type Out = [u8; 32]; + type StdHasher = Hash256StdHasher; + const LENGTH: usize = 32; + + fn hash(x: &[u8]) -> Self::Out { + blake3::hash(x).into() + } + } + + impl BinaryHasher for Blake3Hasher { +/* const NULL_HASH: &'static [u8] = &[14, 29, 19, + 198, 45, 46, 139, 195, 197, 200, 181, 152, 74, + 15, 2, 1, 45, 203, 135, 228, 100, 205, 129, 59, + 26, 116, 168, 125, 248, 104, 37, 44];*/ + const NULL_HASH: &'static [u8] = &[152, 15, 77, 200, + 134, 47, 65, 45, 215, 24, 24, 193, 28, 97, 126, 92, + 216, 199, 142, 241, 18, 148, 20, 118, 174, 171, 136, + 135, 20, 59, 230, 86]; + + + type Buffer = blake3::Hasher; + + fn init_buffer() -> Self::Buffer { + blake3::Hasher::new() + } + + fn reset_buffer(buff: &mut Self::Buffer) { + //let _ = core::mem::replace(buff, Self::init_buffer()); + buff.reset(); + } + + fn buffer_hash(buff: &mut Self::Buffer, x: &[u8]) { + buff.update(&x[..]); + } + + fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out { + buff.finalize().into() + } + } + + #[test] + fn test_blake3_hasher() { + hash_db::test_binary_hasher::() + } +} diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 72206e54..3cbe8dcf 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1226,7 +1226,7 @@ where None, One(u8), Many, - }; + } let mut used_index = UsedIndex::None; for i in 0..16 { match (children[i].is_none(), &used_index) { @@ -1272,7 +1272,7 @@ where None, One(u8), Many, - }; + } let mut used_index = UsedIndex::None; for i in 0..16 { match (children[i].is_none(), &used_index) { From 55dc53c2bb01f07f0ec32f1b11db9f6997922622 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 26 Mar 2021 10:45:15 +0100 Subject: [PATCH 73/73] fix binary hasher --- test-support/reference-trie/src/lib.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index b39f7bb1..a9ad034b 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -54,8 +54,8 @@ pub mod node { } /// Reference hasher is a keccak hasher with hybrid ordered trie implementation. -//pub type RefHasher = ordered_trie::OrderedTrieHasher; -pub type RefHasher = ordered_trie::OrderedTrieHasher; +pub type RefHasher = ordered_trie::OrderedTrieHasher; +//pub type RefHasher = ordered_trie::OrderedTrieHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; //use keccak_hasher::KeccakHasher; //pub type RefHasher = ordered_trie::OrderedTrieHasher; @@ -1734,15 +1734,10 @@ pub mod blake3 { } impl BinaryHasher for Blake3Hasher { -/* const NULL_HASH: &'static [u8] = &[14, 29, 19, - 198, 45, 46, 139, 195, 197, 200, 181, 152, 74, - 15, 2, 1, 45, 203, 135, 228, 100, 205, 129, 59, - 26, 116, 168, 125, 248, 104, 37, 44];*/ - const NULL_HASH: &'static [u8] = &[152, 15, 77, 200, - 134, 47, 65, 45, 215, 24, 24, 193, 28, 97, 126, 92, - 216, 199, 142, 241, 18, 148, 20, 118, 174, 171, 136, - 135, 20, 59, 230, 86]; - + const NULL_HASH: &'static [u8] = &[175, 19, 73, 185, + 245, 249, 161, 166, 160, 64, 77, 234, 54, 220, 201, + 73, 155, 203, 37, 201, 173, 193, 18, 183, 204, 154, + 147, 202, 228, 31, 50, 98]; type Buffer = blake3::Hasher; @@ -1751,7 +1746,6 @@ pub mod blake3 { } fn reset_buffer(buff: &mut Self::Buffer) { - //let _ = core::mem::replace(buff, Self::init_buffer()); buff.reset(); } @@ -1760,7 +1754,10 @@ pub mod blake3 { } fn buffer_finalize(buff: &mut Self::Buffer) -> Self::Out { - buff.finalize().into() + let result = buff.finalize(); + // finalize do not reset state + buff.reset(); + result.into() } }