diff --git a/src/any_list.rs b/src/any_list.rs new file mode 100644 index 0000000..3ecaf1a --- /dev/null +++ b/src/any_list.rs @@ -0,0 +1,322 @@ +//! View types that abstract over `List` and `ProgressiveList`. +//! +//! These are intended for downstream consumers (like Lighthouse's `BeaconState`) where the +//! same field is stored as a `List` in some variants and a `ProgressiveList` in others. + +use crate::interface_iter::{InterfaceIter, InterfaceIterCow}; +use crate::progressive_list::{ProgressiveListIter, ProgressiveListIterCow}; +use crate::update_map::MaxMap; +use crate::{Cow, Error, List, ProgressiveList, UpdateMap, Value}; +use serde::{Serialize, Serializer}; +use typenum::Unsigned; +use vec_map::VecMap; + +/// Immutable view over either a fixed-capacity `List` or a `ProgressiveList`. +/// +/// Methods take `self` by value (the view is `Copy`) and return data tied to the underlying +/// borrow `'a`, so views can be used as temporaries, e.g. `state.validators().iter()`. +#[derive(Debug)] +pub enum AnyListRef<'a, T: Value, N: Unsigned, U: UpdateMap = MaxMap>> { + Basic(&'a List), + Progressive(&'a ProgressiveList), +} + +// Manual `Clone`/`Copy` impls to avoid spurious `T: Copy` bounds from the derive. +impl> Clone for AnyListRef<'_, T, N, U> { + fn clone(&self) -> Self { + *self + } +} + +impl> Copy for AnyListRef<'_, T, N, U> {} + +impl<'a, T: Value, N: Unsigned, U: UpdateMap> AnyListRef<'a, T, N, U> { + pub fn len(self) -> usize { + match self { + Self::Basic(list) => list.len(), + Self::Progressive(list) => list.len(), + } + } + + pub fn is_empty(self) -> bool { + self.len() == 0 + } + + pub fn get(self, index: usize) -> Option<&'a T> { + match self { + Self::Basic(list) => list.get(index), + Self::Progressive(list) => list.get(index), + } + } + + pub fn iter(self) -> AnyListIter<'a, T, U> { + match self { + Self::Basic(list) => AnyListIter::Basic(list.iter()), + Self::Progressive(list) => AnyListIter::Progressive(list.iter()), + } + } + + pub fn iter_from(self, index: usize) -> Result, Error> { + match self { + Self::Basic(list) => list.iter_from(index).map(AnyListIter::Basic), + Self::Progressive(list) => list.iter_from(index).map(AnyListIter::Progressive), + } + } + + pub fn has_pending_updates(self) -> bool { + match self { + Self::Basic(list) => list.has_pending_updates(), + Self::Progressive(list) => list.has_pending_updates(), + } + } + + pub fn to_vec(self) -> Vec { + match self { + Self::Basic(list) => list.to_vec(), + Self::Progressive(list) => list.to_vec(), + } + } + + /// Clone the underlying list into an owned `AnyList`. + pub fn to_owned_list(self) -> AnyList { + match self { + Self::Basic(list) => AnyList::Basic(list.clone()), + Self::Progressive(list) => AnyList::Progressive(list.clone()), + } + } + + /// Compute the tree hash root of the underlying list. + /// + /// NOTE: like `List`/`ProgressiveList`, this is only valid once pending updates have been + /// applied. + pub fn tree_hash_root(self) -> tree_hash::Hash256 + where + T: Send + Sync, + { + use tree_hash::TreeHash; + match self { + Self::Basic(list) => list.tree_hash_root(), + Self::Progressive(list) => list.tree_hash_root(), + } + } +} + +impl> Serialize for AnyListRef<'_, T, N, U> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Self::Basic(list) => list.serialize(serializer), + Self::Progressive(list) => list.serialize(serializer), + } + } +} + +impl<'a, T: Value, N: Unsigned, U: UpdateMap> IntoIterator for AnyListRef<'a, T, N, U> { + type Item = &'a T; + type IntoIter = AnyListIter<'a, T, U>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// Mutable view over either a fixed-capacity `List` or a `ProgressiveList`. +#[derive(Debug)] +pub enum AnyListMut<'a, T: Value, N: Unsigned, U: UpdateMap = MaxMap>> { + Basic(&'a mut List), + Progressive(&'a mut ProgressiveList), +} + +impl<'a, T: Value, N: Unsigned, U: UpdateMap> AnyListMut<'a, T, N, U> { + pub fn as_ref(&self) -> AnyListRef<'_, T, N, U> { + match self { + Self::Basic(list) => AnyListRef::Basic(list), + Self::Progressive(list) => AnyListRef::Progressive(list), + } + } + + pub fn len(&self) -> usize { + self.as_ref().len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get(&self, index: usize) -> Option<&T> { + self.as_ref().get(index) + } + + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + match self { + Self::Basic(list) => list.get_mut(index), + Self::Progressive(list) => list.get_mut(index), + } + } + + pub fn get_cow(&mut self, index: usize) -> Option> { + match self { + Self::Basic(list) => list.get_cow(index), + Self::Progressive(list) => list.get_cow(index), + } + } + + pub fn push(&mut self, value: T) -> Result<(), Error> { + match self { + Self::Basic(list) => list.push(value), + Self::Progressive(list) => list.push(value), + } + } + + pub fn apply_updates(&mut self) -> Result<(), Error> { + match self { + Self::Basic(list) => list.apply_updates(), + Self::Progressive(list) => list.apply_updates(), + } + } + + pub fn has_pending_updates(&self) -> bool { + self.as_ref().has_pending_updates() + } + + pub fn pop_front(&mut self, n: usize) -> Result<(), Error> { + match self { + Self::Basic(list) => list.pop_front(n), + Self::Progressive(list) => list.pop_front(n), + } + } + + pub fn iter_cow(&mut self) -> AnyListIterCow<'_, T, U> { + match self { + Self::Basic(list) => AnyListIterCow::Basic(list.iter_cow()), + Self::Progressive(list) => AnyListIterCow::Progressive(list.iter_cow()), + } + } + + /// Consume the view and return a mutable reference to the element at `index`. + pub fn into_get_mut(self, index: usize) -> Option<&'a mut T> { + match self { + Self::Basic(list) => list.get_mut(index), + Self::Progressive(list) => list.get_mut(index), + } + } + + /// Consume the view and return a CoW reference to the element at `index`. + pub fn into_get_cow(self, index: usize) -> Option> { + match self { + Self::Basic(list) => list.get_cow(index), + Self::Progressive(list) => list.get_cow(index), + } + } +} + +/// Owned list that is either a fixed-capacity `List` or a `ProgressiveList`. +#[derive(Debug, Clone)] +pub enum AnyList = MaxMap>> { + Basic(List), + Progressive(ProgressiveList), +} + +impl> AnyList { + pub fn as_ref(&self) -> AnyListRef<'_, T, N, U> { + match self { + Self::Basic(list) => AnyListRef::Basic(list), + Self::Progressive(list) => AnyListRef::Progressive(list), + } + } + + pub fn as_mut(&mut self) -> AnyListMut<'_, T, N, U> { + match self { + Self::Basic(list) => AnyListMut::Basic(list), + Self::Progressive(list) => AnyListMut::Progressive(list), + } + } + + pub fn get(&self, index: usize) -> Option<&T> { + self.as_ref().get(index) + } + + pub fn iter(&self) -> AnyListIter<'_, T, U> { + self.as_ref().iter() + } + + pub fn to_vec(&self) -> Vec { + self.as_ref().to_vec() + } +} + +impl> Serialize for AnyList { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.as_ref().serialize(serializer) + } +} + +impl> From> for AnyList { + fn from(list: List) -> Self { + Self::Basic(list) + } +} + +impl> From> for AnyList { + fn from(list: ProgressiveList) -> Self { + Self::Progressive(list) + } +} + +impl + PartialEq> PartialEq for AnyList { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Basic(a), Self::Basic(b)) => a == b, + (Self::Progressive(a), Self::Progressive(b)) => a == b, + _ => false, + } + } +} + +/// Iterator over either list type. +#[derive(Debug)] +pub enum AnyListIter<'a, T: Value, U: UpdateMap> { + Basic(InterfaceIter<'a, T, U>), + Progressive(ProgressiveListIter<'a, T, U>), +} + +impl<'a, T: Value, U: UpdateMap> Iterator for AnyListIter<'a, T, U> { + type Item = &'a T; + + fn next(&mut self) -> Option<&'a T> { + match self { + Self::Basic(iter) => iter.next(), + Self::Progressive(iter) => iter.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + match self { + Self::Basic(iter) => iter.size_hint(), + Self::Progressive(iter) => iter.size_hint(), + } + } +} + +impl> ExactSizeIterator for AnyListIter<'_, T, U> {} + +/// Copy-on-write iterator over either list type. +#[derive(Debug)] +pub enum AnyListIterCow<'a, T: Value, U: UpdateMap> { + Basic(InterfaceIterCow<'a, T, U>), + Progressive(ProgressiveListIterCow<'a, T, U>), +} + +impl> AnyListIterCow<'_, T, U> { + pub fn next_cow(&mut self) -> Option<(usize, Cow<'_, T>)> { + match self { + Self::Basic(iter) => iter.next_cow(), + Self::Progressive(iter) => iter.next_cow(), + } + } +} diff --git a/src/context_deserialize.rs b/src/context_deserialize.rs index 7642f68..4c0d9af 100644 --- a/src/context_deserialize.rs +++ b/src/context_deserialize.rs @@ -1,4 +1,4 @@ -use crate::{List, Value, Vector}; +use crate::{List, ProgressiveList, UpdateMap, Value, Vector}; use context_deserialize::ContextDeserialize; use serde::de::Deserializer; use typenum::Unsigned; @@ -43,3 +43,24 @@ where }) } } + +impl<'de, C, T, U> ContextDeserialize<'de, C> for ProgressiveList +where + T: ContextDeserialize<'de, C> + Value, + U: UpdateMap, + C: Clone, +{ + fn context_deserialize(deserializer: D, context: C) -> Result + where + D: Deserializer<'de>, + { + // First deserialize as a Vec. + // This is not the most efficient implementation as it allocates a temporary Vec. In future + // we could write a more performant implementation using `ProgressiveList::builder()`. + let vec = Vec::::context_deserialize(deserializer, context)?; + + // Then convert to List, which will check the length. + ProgressiveList::try_from(vec) + .map_err(|e| serde::de::Error::custom(format!("Failed to create List: {:?}", e))) + } +} diff --git a/src/lib.rs b/src/lib.rs index c09d0fd..bf7f691 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::comparison_chain)] #![deny(clippy::unwrap_used)] +pub mod any_list; pub mod builder; pub mod cow; pub mod error; @@ -12,6 +13,8 @@ pub mod level_iter; pub mod list; pub mod mem; pub mod packed_leaf; +pub mod progressive_list; +pub mod progressive_tree; mod repeat; pub mod serde; mod tests; @@ -23,12 +26,14 @@ pub mod vector; #[cfg(feature = "context_deserialize")] mod context_deserialize; +pub use any_list::{AnyList, AnyListIter, AnyListIterCow, AnyListMut, AnyListRef}; pub use cow::Cow; pub use error::Error; pub use interface::ImmList; pub use leaf::Leaf; pub use list::List; pub use packed_leaf::PackedLeaf; +pub use progressive_list::ProgressiveList; pub use tree::Tree; pub use triomphe::Arc; pub use update_map::UpdateMap; diff --git a/src/progressive_list.rs b/src/progressive_list.rs new file mode 100644 index 0000000..1eff12e --- /dev/null +++ b/src/progressive_list.rs @@ -0,0 +1,485 @@ +use crate::{ + Arc, Cow, Error, UpdateMap, Value, + progressive_tree::{ProgressiveTree, ProgressiveTreeIter}, + update_map::MaxMap, + utils::{Length, updated_length}, +}; +use educe::Educe; +use itertools::process_results; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _, ser::SerializeSeq}; +use ssz::{BYTES_PER_LENGTH_OFFSET, Decode, Encode, SszEncoder, TryFromIter}; +use std::convert::TryFrom; +use std::ops::ControlFlow; +use tree_hash::{Hash256, PackedEncoding, TreeHash}; +use vec_map::VecMap; + +#[derive(Debug, Clone, Educe)] +#[educe(PartialEq(bound(T: Value, U: UpdateMap + PartialEq)))] +pub struct ProgressiveList = MaxMap>> { + pub(crate) tree: Arc>, + pub(crate) length: Length, + pub(crate) updates: U, +} + +impl> ProgressiveList { + pub fn empty() -> Self { + Self { + tree: Arc::new(ProgressiveTree::empty()), + length: Length(0), + updates: U::default(), + } + } + + pub fn new(vec: Vec) -> Result { + Self::try_from_iter(vec) + } + + pub fn try_from_iter(iter: impl IntoIterator) -> Result { + let items: Vec = iter.into_iter().collect(); + let length = items.len(); + let tree = ProgressiveTree::build_from_iter(items)?; + Ok(Self { + tree: Arc::new(tree), + length: Length(length), + updates: U::default(), + }) + } + + /// The length of the backing tree, ignoring any pending updates. + fn backing_len(&self) -> usize { + self.length.as_usize() + } + + /// Get the value at `index` from the backing tree only (ignoring pending updates). + fn backing_get(&self, index: usize) -> Option<&T> { + if index < self.backing_len() { + self.tree.get_recursive(index, 0) + } else { + None + } + } + + pub fn get(&self, index: usize) -> Option<&T> { + self.updates.get(index).or_else(|| self.backing_get(index)) + } + + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + self.updates.get_mut_with(index, |index| { + if index < self.length.as_usize() { + self.tree.get_recursive(index, 0).cloned() + } else { + None + } + }) + } + + pub fn get_cow(&mut self, index: usize) -> Option> { + self.updates.get_cow_with(index, |index| { + if index < self.length.as_usize() { + self.tree.get_recursive(index, 0) + } else { + None + } + }) + } + + pub fn push(&mut self, value: T) -> Result<(), Error> { + let index = self.len(); + self.updates.insert(index, value); + Ok(()) + } + + pub fn len(&self) -> usize { + updated_length(self.length, &self.updates).as_usize() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn has_pending_updates(&self) -> bool { + !self.updates.is_empty() + } + + /// Apply all pending updates to the backing tree. + /// + /// Updates are applied in ascending index order, which means appends (indices `>= length`) + /// land at the end of the list one at a time, while replaces (indices `< length`) update + /// existing leaves in place. The list never has gaps, so this ordering is well-defined. + pub fn apply_updates(&mut self) -> Result<(), Error> { + if self.updates.is_empty() { + return Ok(()); + } + + let updates = std::mem::take(&mut self.updates); + let new_length = updated_length(self.length, &updates); + + let mut tree = self.tree.clone(); + let mut running_length = self.length.as_usize(); + + updates.for_each_range(0, new_length.as_usize(), |index, value| { + match tree.with_updated_leaf(index, value.clone(), running_length) { + Ok(new_tree) => { + tree = Arc::new(new_tree); + if index >= running_length { + running_length = index + 1; + } + ControlFlow::Continue(Ok(())) + } + Err(e) => ControlFlow::Continue(Err(e)), + } + })?; + + self.tree = tree; + self.length = new_length; + Ok(()) + } + + pub fn bulk_update(&mut self, updates: U) -> Result<(), Error> { + if !self.updates.is_empty() { + return Err(Error::BulkUpdateUnclean); + } + self.updates = updates; + Ok(()) + } + + pub fn iter(&self) -> ProgressiveListIter<'_, T, U> { + self.iter_from_unchecked(0) + } + + pub fn iter_from(&self, index: usize) -> Result, Error> { + // Return an empty iterator at `index == len`, just like slicing. + if index > self.len() { + return Err(Error::OutOfBoundsIterFrom { + index, + len: self.len(), + }); + } + Ok(self.iter_from_unchecked(index)) + } + + fn iter_from_unchecked(&self, index: usize) -> ProgressiveListIter<'_, T, U> { + let backing_len = self.backing_len(); + let mut tree_iter = self.tree.iter(backing_len); + + // Advance the backing iterator so it stays in step with the merged index. + for _ in 0..index.min(backing_len) { + tree_iter.next(); + } + + ProgressiveListIter { + tree_iter, + updates: &self.updates, + index, + length: self.len(), + } + } + + pub fn iter_cow(&mut self) -> ProgressiveListIterCow<'_, T, U> { + self.iter_cow_from_unchecked(0) + } + + pub fn iter_cow_from( + &mut self, + index: usize, + ) -> Result, Error> { + if index > self.len() { + return Err(Error::OutOfBoundsIterFrom { + index, + len: self.len(), + }); + } + Ok(self.iter_cow_from_unchecked(index)) + } + + fn iter_cow_from_unchecked(&mut self, index: usize) -> ProgressiveListIterCow<'_, T, U> { + let backing_len = self.backing_len(); + let mut tree_iter = self.tree.iter(backing_len); + + for _ in 0..index.min(backing_len) { + tree_iter.next(); + } + + ProgressiveListIterCow { + tree_iter, + updates: &mut self.updates, + index, + } + } + + pub fn to_vec(&self) -> Vec { + self.iter().cloned().collect() + } + + /// Remove `n` elements from the front of `self`. + /// + /// Errors if `n > self.len()`. + pub fn pop_front(&mut self, n: usize) -> Result<(), Error> { + self.apply_updates()?; + + if n == 0 { + return Ok(()); + } + if n > self.len() { + return Err(Error::OutOfBoundsIterFrom { + index: n, + len: self.len(), + }); + } + + // The progressive structure re-indexes on removal, so rebuild from the remaining elements. + let remaining: Vec = self.iter_from(n)?.cloned().collect(); + *self = Self::try_from_iter(remaining)?; + + Ok(()) + } +} + +impl> ProgressiveList { + pub fn rebase(&self, base: &Self) -> Result { + let mut rebased = self.clone(); + rebased.rebase_on(base)?; + Ok(rebased) + } + + /// Rebase `self` onto `base`, sharing memory between equal subtrees. Pending updates on `self` + /// are applied first; `base` is used as-is. + pub fn rebase_on(&mut self, base: &Self) -> Result<(), Error> { + self.apply_updates()?; + + self.tree = ProgressiveTree::rebase_on( + &self.tree, + &base.tree, + self.length.as_usize(), + base.length.as_usize(), + )?; + + Ok(()) + } +} + +impl> TryFrom> for ProgressiveList { + type Error = Error; + + fn try_from(vec: Vec) -> Result { + Self::try_from_iter(vec) + } +} + +impl> Default for ProgressiveList { + fn default() -> Self { + Self::empty() + } +} + +impl<'a, T: Value, U: UpdateMap> IntoIterator for &'a ProgressiveList { + type Item = &'a T; + type IntoIter = ProgressiveListIter<'a, T, U>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl> TreeHash for ProgressiveList { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> PackedEncoding { + unreachable!("ProgressiveList should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("ProgressiveList should never be packed.") + } + + fn tree_hash_root(&self) -> Hash256 { + assert!( + !self.has_pending_updates(), + "ProgressiveList has pending updates at tree_hash_root" + ); + + let root = self.tree.tree_hash(); + tree_hash::mix_in_length(&root, self.len()) + } +} + +impl> Serialize for ProgressiveList { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.len()))?; + for e in self { + seq.serialize_element(e)?; + } + seq.end() + } +} + +// FIXME: duplicated from `ssz::encode::impl_for_vec` +impl> Encode for ProgressiveList { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() * self.len() + } else { + let mut len = self.iter().map(|item| item.ssz_bytes_len()).sum(); + len += BYTES_PER_LENGTH_OFFSET * self.len(); + len + } + } + + fn ssz_append(&self, buf: &mut Vec) { + if ::is_ssz_fixed_len() { + buf.reserve(::ssz_fixed_len() * self.len()); + + for item in self { + item.ssz_append(buf); + } + } else { + let mut encoder = SszEncoder::container(buf, self.len() * BYTES_PER_LENGTH_OFFSET); + + for item in self { + encoder.append(item); + } + + encoder.finalize(); + } + } +} + +impl TryFromIter for ProgressiveList +where + T: Value, + U: UpdateMap, +{ + type Error = Error; + + fn try_from_iter(iter: I) -> Result + where + I: IntoIterator, + { + ProgressiveList::try_from_iter(iter) + } +} + +impl Decode for ProgressiveList +where + T: Value, + U: UpdateMap, +{ + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + if bytes.is_empty() { + Ok(ProgressiveList::empty()) + } else if ::is_ssz_fixed_len() { + process_results( + bytes + .chunks(::ssz_fixed_len()) + .map(T::from_ssz_bytes), + |iter| { + ProgressiveList::try_from_iter(iter).map_err(|e| { + ssz::DecodeError::BytesInvalid(format!( + "Error building ssz ProgressiveList: {e:?}" + )) + }) + }, + )? + } else { + ssz::decode_list_of_variable_length_items(bytes, None) + } + } +} + +impl<'de, T, U> Deserialize<'de> for ProgressiveList +where + T: Deserialize<'de> + Value, + U: UpdateMap, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // TODO: this implementation is not necessarily the most efficient + Self::try_from_iter(Vec::deserialize(deserializer)?) + .map_err(|e| D::Error::custom(format!("{e:?}"))) + } +} + +#[cfg(feature = "arbitrary")] +impl<'a, T, U> arbitrary::Arbitrary<'a> for ProgressiveList +where + T: arbitrary::Arbitrary<'a> + Value, + U: UpdateMap, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + // Build from a `Vec` so the backing tree and length are always consistent. + let vec = Vec::::arbitrary(u)?; + Self::new(vec).map_err(|_| arbitrary::Error::IncorrectFormat) + } +} + +/// Iterator over a [`ProgressiveList`] that merges the backing tree with pending updates. +#[derive(Debug)] +pub struct ProgressiveListIter<'a, T: Value, U: UpdateMap> { + tree_iter: ProgressiveTreeIter<'a, T>, + updates: &'a U, + index: usize, + length: usize, +} + +impl<'a, T: Value, U: UpdateMap> Iterator for ProgressiveListIter<'a, T, U> { + type Item = &'a T; + + fn next(&mut self) -> Option<&'a T> { + if self.index >= self.length { + return None; + } + + let index = self.index; + self.index += 1; + + // Advance the tree iterator so that it moves in step with this iterator. + let backing_value = self.tree_iter.next(); + + // Prioritise the value from the update map. + self.updates.get(index).or(backing_value) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.length.saturating_sub(self.index); + (remaining, Some(remaining)) + } +} + +impl> ExactSizeIterator for ProgressiveListIter<'_, T, U> {} + +#[derive(Debug)] +pub struct ProgressiveListIterCow<'a, T: Value, U: UpdateMap> { + tree_iter: ProgressiveTreeIter<'a, T>, + updates: &'a mut U, + index: usize, +} + +impl> ProgressiveListIterCow<'_, T, U> { + pub fn next_cow(&mut self) -> Option<(usize, Cow<'_, T>)> { + let index = self.index; + self.index += 1; + + // Advance the tree iterator so that it moves in step with this iterator. + let backing_value = self.tree_iter.next(); + + // Construct a CoW pointer using the updated entry from the map, or the corresponding + // vacant entry and the value from the backing iterator. + let cow = self.updates.get_cow_with(index, |_| backing_value)?; + Some((index, cow)) + } +} diff --git a/src/progressive_tree.rs b/src/progressive_tree.rs new file mode 100644 index 0000000..84e6ffa --- /dev/null +++ b/src/progressive_tree.rs @@ -0,0 +1,513 @@ +use crate::{ + Arc, Error, Tree, Value, + builder::Builder, + iter::Iter, + tree::RebaseAction, + utils::{Length, opt_packing_depth, opt_packing_factor}, +}; +use educe::Educe; +use ethereum_hashing::hash32_concat; +use parking_lot::RwLock; +use tree_hash::Hash256; + +/// The size of each binary subtree in a progressive tree is `4^prog_depth` at depth `prog_depth`. +const PROG_TREE_EXPONENT: usize = 4; + +/// This scaling factor is used to convert between a 4-based progressive depth and a 2-based +/// depth for a binary subtree. +/// +/// It is defined such that the binary subtree at progressive depth `prog_depth` has depth +/// `PROG_TREE_BINARY_SCALE * prog_depth`. This comes from this equation: +/// +/// PROG_TREE_EXPONENT^prog_depth = 2^binary_depth +/// +/// Hence: +/// +/// binary_depth = log2(PROG_TREE_EXPONENT^prog_depth) +/// +/// Knowing PROG_TREE_EXPONENT is `2^k` for some `k`, this becomes: +/// +/// binary_depth = log2(2^(k * prog_depth)) +/// = k * prog_depth +/// +/// This `k` is the scaling factor, equal to `log2(PROG_TREE_EXPONENT)`. +const PROG_TREE_BINARY_SCALE: usize = PROG_TREE_EXPONENT.trailing_zeros() as usize; + +/// Tree type for the implementation of `ProgressiveList`. +#[derive(Debug, Educe)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[educe(PartialEq(bound(T: Value)), Hash)] +pub enum ProgressiveTree { + ProgressiveZero, + ProgressiveNode { + #[educe(PartialEq(ignore), Hash(ignore))] + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::utils::arb_rwlock))] + hash: RwLock, + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::utils::arb_arc))] + left: Arc>, + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::utils::arb_arc))] + right: Arc, + }, +} + +impl Clone for ProgressiveTree { + fn clone(&self) -> Self { + match self { + Self::ProgressiveNode { hash, left, right } => Self::ProgressiveNode { + hash: RwLock::new(*hash.read()), + left: left.clone(), + right: right.clone(), + }, + Self::ProgressiveZero => Self::ProgressiveZero, + } + } +} + +impl ProgressiveTree { + pub fn empty() -> Self { + Self::ProgressiveZero + } + + /// The number of values that can be stored in the single subtree at `prog_depth` itself. + pub fn capacity_at_depth(prog_depth: u32) -> usize { + let capacity_pre_packing = match prog_depth.checked_sub(1) { + None => 0, + Some(depth_minus_one) => PROG_TREE_EXPONENT.pow(depth_minus_one), + }; + capacity_pre_packing * opt_packing_factor::().unwrap_or(1) + } + + /// The number of values that be stored in the whole progressive tree up to and including + /// the layer at `prog_depth`. + pub fn total_capacity_at_depth(prog_depth: u32) -> usize { + let total_capacity_pre_packing = + PROG_TREE_EXPONENT.pow(prog_depth).saturating_sub(1) / (PROG_TREE_EXPONENT - 1); + total_capacity_pre_packing * opt_packing_factor::().unwrap_or(1) + } + + /// Calculate the depth for the binary subtree at `prog_depth`. + pub fn prog_depth_to_binary_depth(prog_depth: u32) -> usize { + match prog_depth.checked_sub(1) { + None => 0, + Some(prog_depth_minus_one) => { + // FIXME: work out why we don't need to sub the packing depth here, seems weird + PROG_TREE_BINARY_SCALE * prog_depth_minus_one as usize + } + } + } + + /// Build a `ProgressiveTree` efficiently from an iterator. + /// Only creates one `Builder` per subtree. + pub fn build_from_iter(iter: impl IntoIterator) -> Result { + let mut iter = iter.into_iter(); + let mut subtrees: Vec>> = Vec::new(); + let mut prog_depth = 1u32; + + loop { + let capacity = Self::capacity_at_depth(prog_depth); + let binary_depth = Self::prog_depth_to_binary_depth(prog_depth); + + let mut builder = Builder::::new(binary_depth, 0)?; + let mut count = 0; + + // Fill up each subtree to its capacity. + while count < capacity { + match iter.next() { + Some(item) => { + builder.push(item)?; + count += 1; + } + None => break, + } + } + + if count == 0 { + // No items left. + break; + } + + let (tree, _, _) = builder.finish()?; + subtrees.push(tree); + + if count < capacity { + // No items left and subtree is only partially filled. + break; + } + + // Move to the next subtree. + prog_depth += 1; + } + + // Assemble the `ProgressiveTree` in reverse from deepest to shallowest. + let mut current = Self::ProgressiveZero; + for tree in subtrees.into_iter().rev() { + current = Self::ProgressiveNode { + hash: RwLock::new(Hash256::ZERO), + left: tree, + right: Arc::new(current), + }; + } + + Ok(current) + } + + fn push_recursive( + &self, + value: T, + current_length: usize, + prog_depth: u32, + ) -> Result { + match self { + // Expand this zero into a new left node for our element. + Self::ProgressiveZero => { + // The `prog_depth` of the new left subtree is `prog_depth + 1`. + let subtree_depth = Self::prog_depth_to_binary_depth(prog_depth + 1); + let mut tree_builder = Builder::::new(subtree_depth, 0)?; + tree_builder.push(value)?; + let (new_left, _, _) = tree_builder.finish()?; + + Ok(Self::ProgressiveNode { + hash: RwLock::new(Hash256::ZERO), + left: new_left, + right: Arc::new(Self::ProgressiveZero), + }) + } + Self::ProgressiveNode { + hash: _, + left, + right, + } => { + // Case 1: new element already fits inside the left-tree at prog_depth + 1. + let total_capacity_at_depth = Self::total_capacity_at_depth(prog_depth + 1); + if current_length < total_capacity_at_depth { + let index = + current_length.saturating_sub(Self::total_capacity_at_depth(prog_depth)); + + // Our left subtree can hold 4^prog_depth entries. We need to work out + // a 2-based depth for this sub tree, such that the subtree holds + // 2^subtree_depth entries. + let subtree_depth = Self::prog_depth_to_binary_depth(prog_depth + 1); + let new_left = left.with_updated_leaf(index, value, subtree_depth)?; + + // FIXME: remove assert + debug_assert!(matches!(**right, Self::ProgressiveZero)); + + Ok(Self::ProgressiveNode { + hash: RwLock::new(Hash256::ZERO), + left: new_left, + right: right.clone(), + }) + } else { + // Case 2: new element does not fit inside this left-tree: recurse to the next + // level on the right. + let new_right = right.push_recursive(value, current_length, prog_depth + 1)?; + + Ok(Self::ProgressiveNode { + hash: RwLock::new(Hash256::ZERO), + left: left.clone(), + right: Arc::new(new_right), + }) + } + } + } + } + + pub fn push(&self, value: T, current_length: usize) -> Result { + self.push_recursive(value, current_length, 0) + } + + /// Create an iterator over all elements in the progressive tree. + /// + /// The iterator traverses elements in order by visiting each binary subtree + /// (left child) at increasing progressive depths: + /// 1. All elements in the left child at the root level + /// 2. All elements in the left child of the first right node + /// 3. All elements in the left child of the second right node + /// + /// And so on, following the progressive tree structure as defined in EIP-7916. + pub fn iter(&self, length: usize) -> ProgressiveTreeIter<'_, T> { + ProgressiveTreeIter::new(self, length) + } + + pub fn get_recursive(&self, index: usize, prog_depth: u32) -> Option<&T> { + match self { + Self::ProgressiveZero => None, + Self::ProgressiveNode { left, right, .. } => { + let total_capacity = Self::total_capacity_at_depth(prog_depth + 1); + if index < total_capacity { + // Index is in the left subtree (binary tree). + let subtree_index = + index.saturating_sub(Self::total_capacity_at_depth(prog_depth)); + let binary_depth = Self::prog_depth_to_binary_depth(prog_depth + 1); + let packing_depth = opt_packing_depth::().unwrap_or(0); + left.get_recursive(subtree_index, binary_depth, packing_depth) + } else { + // Index is in the right subtree (progressive tree). + right.get_recursive(index, prog_depth + 1) + } + } + } + } + + /// Create a new tree where the `index`th leaf is set to `value`. + /// + /// If `index == current_length` the value is appended (equivalent to `push`). Updating any + /// index `> current_length` is an error, as it would leave a gap in the list. + pub fn with_updated_leaf( + &self, + index: usize, + value: T, + current_length: usize, + ) -> Result { + if index < current_length { + self.replace_recursive(index, value, 0) + } else if index == current_length { + self.push_recursive(value, current_length, 0) + } else { + Err(Error::OutOfBoundsUpdate { + index, + len: current_length, + }) + } + } + + /// Replace the value at an *existing* `index`, leaving the list length unchanged. + /// + /// Routes to the binary subtree containing `index` using the same geometry as + /// `get_recursive`/`push_recursive`, then delegates the in-subtree update to `Tree`. + fn replace_recursive(&self, index: usize, value: T, prog_depth: u32) -> Result { + match self { + Self::ProgressiveZero => Err(Error::OutOfBoundsUpdate { index, len: 0 }), + Self::ProgressiveNode { left, right, .. } => { + let total_capacity = Self::total_capacity_at_depth(prog_depth + 1); + if index < total_capacity { + // Index lies in this node's binary (left) subtree. + let subtree_index = + index.saturating_sub(Self::total_capacity_at_depth(prog_depth)); + let binary_depth = Self::prog_depth_to_binary_depth(prog_depth + 1); + let new_left = left.with_updated_leaf(subtree_index, value, binary_depth)?; + + Ok(Self::ProgressiveNode { + hash: RwLock::new(Hash256::ZERO), + left: new_left, + right: right.clone(), + }) + } else { + // Recurse into the right (progressive) subtree. + let new_right = right.replace_recursive(index, value, prog_depth + 1)?; + + Ok(Self::ProgressiveNode { + hash: RwLock::new(Hash256::ZERO), + left: left.clone(), + right: Arc::new(new_right), + }) + } + } + } + } + + /// Rebase `orig` onto `base`, exploiting structural sharing between equal subtrees to reduce + /// memory usage. The logical contents and `tree_hash` of `orig` are unchanged; only `Arc` + /// pointers are shared with `base` where subtrees are equal. + pub fn rebase_on( + orig: &Arc, + base: &Arc, + orig_length: usize, + base_length: usize, + ) -> Result, Error> { + Self::rebase_on_recursive(orig, base, orig_length, base_length, 0) + } + + fn rebase_on_recursive( + orig: &Arc, + base: &Arc, + orig_length: usize, + base_length: usize, + prog_depth: u32, + ) -> Result, Error> { + if Arc::ptr_eq(orig, base) { + return Ok(base.clone()); + } + match (&**orig, &**base) { + // `orig` is empty here, or `base` has nothing to share. Keep `orig` as-is. + (Self::ProgressiveZero, _) | (Self::ProgressiveNode { .. }, Self::ProgressiveZero) => { + Ok(orig.clone()) + } + ( + Self::ProgressiveNode { + hash: orig_hash, + left: orig_left, + right: orig_right, + }, + Self::ProgressiveNode { + left: base_left, + right: base_right, + .. + }, + ) => { + let lo = Self::total_capacity_at_depth(prog_depth); + let subtree_capacity = Self::capacity_at_depth(prog_depth + 1); + let binary_depth = Self::prog_depth_to_binary_depth(prog_depth + 1); + let packing_depth = opt_packing_depth::().unwrap_or(0); + + let orig_left_len = orig_length.saturating_sub(lo).min(subtree_capacity); + let base_left_len = base_length.saturating_sub(lo).min(subtree_capacity); + + // Rebase the binary (left) subtree using the existing `Tree` machinery. + let new_left = match Tree::rebase_on( + orig_left, + base_left, + Some((Length(orig_left_len), Length(base_left_len))), + binary_depth + packing_depth, + )? { + RebaseAction::EqualReplace(replacement) => replacement.clone(), + RebaseAction::NotEqualReplace(replacement) => replacement, + RebaseAction::EqualNoop | RebaseAction::NotEqualNoop => orig_left.clone(), + }; + + // Rebase the progressive (right) subtree. + let new_right = Self::rebase_on_recursive( + orig_right, + base_right, + orig_length, + base_length, + prog_depth + 1, + )?; + + // Avoid allocating a new node if nothing changed. + if Arc::ptr_eq(&new_left, orig_left) && Arc::ptr_eq(&new_right, orig_right) { + Ok(orig.clone()) + } else { + Ok(Arc::new(Self::ProgressiveNode { + hash: RwLock::new(*orig_hash.read()), + left: new_left, + right: new_right, + })) + } + } + } + } +} + +impl ProgressiveTree { + pub fn tree_hash(&self) -> Hash256 { + match self { + Self::ProgressiveZero => Hash256::ZERO, + Self::ProgressiveNode { hash, left, right } => { + let read_lock = hash.read(); + let existing_hash = *read_lock; + drop(read_lock); + + if !existing_hash.is_zero() { + existing_hash + } else { + // Parallelism goes brrrr. + let (left_hash, right_hash) = + rayon::join(|| left.tree_hash(), || right.tree_hash()); + let tree_hash = + Hash256::from(hash32_concat(left_hash.as_slice(), right_hash.as_slice())); + *hash.write() = tree_hash; + tree_hash + } + } + } + } +} + +/// Iterator over elements in a progressive tree. +/// +/// The iterator traverses each binary subtree (left child) in sequence by following +/// the right spine of the progressive tree structure. +#[derive(Debug)] +pub struct ProgressiveTreeIter<'a, T: Value> { + /// Current progressive node being traversed. + current_prog_node: Option<&'a ProgressiveTree>, + /// Current iterator over a binary subtree (Tree). + current_iter: Option>, + /// Progressive depth for calculating the next subtree depth. + prog_depth: u32, + /// Total number of elements to iterate. + length: usize, + /// Number of elements already yielded. + yielded: usize, +} + +impl<'a, T: Value> ProgressiveTreeIter<'a, T> { + fn new(root: &'a ProgressiveTree, length: usize) -> Self { + let mut iter = Self { + current_prog_node: Some(root), + current_iter: None, + prog_depth: 0, + length, + yielded: 0, + }; + + // Initialize by setting up the iterator for the first left child + iter.advance_to_next_subtree(); + iter + } + + /// Advance to the next binary subtree by moving to the right child and + /// setting up an iterator for its left child. + fn advance_to_next_subtree(&mut self) { + match self.current_prog_node { + None | Some(ProgressiveTree::ProgressiveZero) => { + // No more subtrees + self.current_iter = None; + self.current_prog_node = None; + } + Some(ProgressiveTree::ProgressiveNode { left, right, .. }) => { + self.prog_depth += 1; + + // Calculate the depth and length for this binary subtree + let binary_depth = + ProgressiveTree::::prog_depth_to_binary_depth(self.prog_depth); + let remaining = self.length.saturating_sub(self.yielded); + let capacity = ProgressiveTree::::capacity_at_depth(self.prog_depth); + let subtree_length = remaining.min(capacity); + + // Create an iterator for the left subtree + self.current_iter = Some(Iter::from_index( + 0, + left, + binary_depth, + Length(subtree_length), + )); + + // Move to the right child for the next iteration + self.current_prog_node = Some(right); + } + } + } +} + +impl<'a, T: Value> Iterator for ProgressiveTreeIter<'a, T> { + type Item = &'a T; + + fn next(&mut self) -> Option { + loop { + // Try to get the next item from the current binary tree iterator + if let Some(iter) = &mut self.current_iter + && let Some(value) = iter.next() + { + self.yielded += 1; + return Some(value); + } + + // Current subtree exhausted, move to the next one + if self.current_prog_node.is_some() { + self.advance_to_next_subtree(); + } else { + // No more subtrees to iterate + return None; + } + } + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.length.saturating_sub(self.yielded); + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for ProgressiveTreeIter<'_, T> {} diff --git a/src/tests/any_list.rs b/src/tests/any_list.rs new file mode 100644 index 0000000..f7c626a --- /dev/null +++ b/src/tests/any_list.rs @@ -0,0 +1,117 @@ +//! Tests for the `AnyList` view types. + +use crate::{AnyList, AnyListMut, AnyListRef, List, ProgressiveList}; +use typenum::U128; + +type BasicList = List; +type ProgList = ProgressiveList; + +fn views(n: usize) -> (BasicList, ProgList) { + let vec: Vec = (0..n as u64).collect(); + ( + BasicList::new(vec.clone()).unwrap(), + ProgList::new(vec).unwrap(), + ) +} + +#[test] +fn ref_views_agree() { + let (basic, prog) = views(20); + let views = [ + AnyListRef::::Basic(&basic), + AnyListRef::::Progressive(&prog), + ]; + + for view in views { + assert_eq!(view.len(), 20); + assert!(!view.is_empty()); + assert_eq!(view.get(7), Some(&7)); + assert_eq!(view.get(20), None); + assert_eq!(view.iter().copied().collect::>(), basic.to_vec()); + assert_eq!( + view.iter_from(15).unwrap().copied().collect::>(), + vec![15, 16, 17, 18, 19] + ); + assert_eq!(view.to_vec(), basic.to_vec()); + // Copy semantics: using the view twice is fine. + assert_eq!(view.len(), view.iter().count()); + } +} + +#[test] +fn mut_views_agree() { + let (mut basic, mut prog) = views(10); + + { + let muts = [ + AnyListMut::::Basic(&mut basic), + AnyListMut::::Progressive(&mut prog), + ]; + + for mut view in muts { + *view.get_mut(3).unwrap() = 333; + view.push(10).unwrap(); + + { + let cow = view.get_cow(4).unwrap(); + *cow.into_mut().unwrap() = 444; + } + + { + let mut iter = view.iter_cow(); + while let Some((index, cow)) = iter.next_cow() { + if index == 5 { + *cow.into_mut().unwrap() = 555; + } + } + } + + assert!(view.has_pending_updates()); + view.apply_updates().unwrap(); + assert!(!view.has_pending_updates()); + + assert_eq!(view.len(), 11); + assert_eq!(view.get(3), Some(&333)); + assert_eq!(view.get(4), Some(&444)); + assert_eq!(view.get(5), Some(&555)); + assert_eq!(view.get(10), Some(&10)); + + view.pop_front(2).unwrap(); + assert_eq!(view.len(), 9); + assert_eq!(view.get(0), Some(&2)); + } + } + + // Both underlying lists ended up identical. + assert_eq!(basic.to_vec(), prog.to_vec()); +} + +#[test] +fn owned_views() { + let (basic, prog) = views(5); + + let owned_basic: AnyList = basic.clone().into(); + let owned_prog: AnyList = prog.into(); + + assert_eq!(owned_basic.to_vec(), vec![0, 1, 2, 3, 4]); + assert_eq!(owned_basic.to_vec(), owned_prog.to_vec()); + assert_eq!(owned_basic.get(2), Some(&2)); + assert_eq!(owned_prog.iter().count(), 5); + + // Same-arm equality works; mixed arms are unequal by design. + assert_eq!(owned_basic, AnyList::Basic(basic.clone())); + assert_ne!(owned_basic, owned_prog); + + // Round-trip through as_ref/to_owned_list. + let reowned = owned_prog.as_ref().to_owned_list(); + assert_eq!(reowned, owned_prog); + + // Mutation through as_mut. + let mut owned = owned_basic; + { + let mut view = owned.as_mut(); + *view.get_mut(0).unwrap() = 100; + view.apply_updates().unwrap(); + } + assert_eq!(owned.get(0), Some(&100)); +} diff --git a/src/tests/mod.rs b/src/tests/mod.rs index b6010f3..e803c55 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,10 +1,13 @@ #![cfg(test)] +mod any_list; mod builder; mod iterator; mod mem; mod packed; mod pop_front; +mod prog; +mod prog_list; mod proptest; mod repeat; mod serde; diff --git a/src/tests/prog.rs b/src/tests/prog.rs new file mode 100644 index 0000000..39af767 --- /dev/null +++ b/src/tests/prog.rs @@ -0,0 +1,107 @@ +use crate::progressive_tree::ProgressiveTree; +use tree_hash::Hash256; + +#[test] +fn wow() { + let empty = ProgressiveTree::::empty(); + + let one = empty.push(Hash256::repeat_byte(0x11), 0).unwrap(); + + let two = one.push(Hash256::repeat_byte(0x22), 1).unwrap(); + + println!("{two:#?}"); + + let three = two.push(Hash256::repeat_byte(0x33), 2).unwrap(); + + println!("{three:#?}"); +} + +#[test] +fn wow_u64() { + let mut tree = ProgressiveTree::::empty(); + + for i in 1..=65 { + tree = tree.push(i, i as usize - 1).unwrap(); + } + + println!("{tree:#?}"); +} + +#[test] +fn prog_tree_iterator() { + let mut tree = ProgressiveTree::::empty(); + + // Build a tree with 65 elements + for i in 1..=65 { + tree = tree.push(i, i as usize - 1).unwrap(); + } + + // Iterate and collect all elements + let collected: Vec<_> = tree.iter(65).copied().collect(); + + // Verify we got all 65 elements in order + assert_eq!(collected.len(), 65); + for (i, &value) in collected.iter().enumerate() { + assert_eq!( + value, + (i + 1) as u64, + "Element at index {} should be {}", + i, + i + 1 + ); + } +} + +#[test] +fn prog_tree_iterator_empty() { + let tree = ProgressiveTree::::empty(); + let collected: Vec<_> = tree.iter(0).collect(); + assert_eq!(collected.len(), 0); +} + +#[test] +fn prog_tree_iterator_small() { + let mut tree = ProgressiveTree::::empty(); + + // Build a small tree with just 4 elements (one packed leaf) + for i in 1..=4 { + tree = tree.push(i, i as usize - 1).unwrap(); + } + + let collected: Vec<_> = tree.iter(4).copied().collect(); + assert_eq!(collected, vec![1, 2, 3, 4]); +} + +#[test] +fn prog_tree_iterator_exact_size() { + let mut tree = ProgressiveTree::::empty(); + + for i in 1..=20 { + tree = tree.push(i, i as usize - 1).unwrap(); + } + + let iter = tree.iter(20); + assert_eq!(iter.len(), 20); + + let collected: Vec<_> = iter.copied().collect(); + assert_eq!(collected.len(), 20); +} + +#[test] +fn prog_tree_iterator_hash256() { + let mut tree = ProgressiveTree::::empty(); + + // Build a tree with non-packed values + for i in 1..=10 { + let hash = Hash256::repeat_byte(i as u8); + tree = tree.push(hash, i - 1).unwrap(); + } + + let collected: Vec<_> = tree.iter(10).collect(); + assert_eq!(collected.len(), 10); + + // Verify order + for (i, hash) in collected.iter().enumerate() { + assert_eq!(**hash, Hash256::repeat_byte((i + 1) as u8)); + } +} diff --git a/src/tests/prog_list.rs b/src/tests/prog_list.rs new file mode 100644 index 0000000..451543b --- /dev/null +++ b/src/tests/prog_list.rs @@ -0,0 +1,280 @@ +//! Tests for the mutation and rebase API of `ProgressiveList`. + +use crate::progressive_tree::ProgressiveTree; +use crate::{Arc, ProgressiveList, Tree}; +use tree_hash::{Hash256, TreeHash}; + +/// Lengths spanning several progressive subtrees and packing boundaries. +const TEST_LENGTHS: &[usize] = &[0, 1, 2, 3, 4, 5, 8, 16, 17, 20, 21, 64, 65, 100, 129]; + +fn build(n: usize) -> ProgressiveList { + ProgressiveList::new((0..n as u64).collect()).unwrap() +} + +/// Return the first (left) binary subtree of a progressive tree. +fn first_left(tree: &ProgressiveTree) -> Option<&Arc>> { + match tree { + ProgressiveTree::ProgressiveZero => None, + ProgressiveTree::ProgressiveNode { left, .. } => Some(left), + } +} + +#[test] +fn push_then_apply_matches_fresh() { + for &n in TEST_LENGTHS { + let mut list = ProgressiveList::::empty(); + for i in 0..n as u64 { + list.push(i).unwrap(); + } + + // Length reflects pending pushes immediately. + assert_eq!(list.len(), n); + assert_eq!( + list.get(n.saturating_sub(1)).copied(), + n.checked_sub(1).map(|x| x as u64) + ); + + list.apply_updates().unwrap(); + + let expected = build(n); + assert_eq!(list.to_vec(), expected.to_vec()); + assert_eq!(list.tree_hash_root(), expected.tree_hash_root()); + } +} + +#[test] +fn get_mut_matches_fresh() { + for &n in TEST_LENGTHS { + if n == 0 { + continue; + } + for &i in &[0usize, n / 2, n - 1] { + let mut list = build(n); + + *list.get_mut(i).unwrap() = 1000 + i as u64; + // Mutation visible before applying. + assert_eq!(list.get(i).copied(), Some(1000 + i as u64)); + assert!(list.has_pending_updates()); + + list.apply_updates().unwrap(); + assert!(!list.has_pending_updates()); + + let mut expected_vec: Vec = (0..n as u64).collect(); + expected_vec[i] = 1000 + i as u64; + let expected = ProgressiveList::::new(expected_vec).unwrap(); + + assert_eq!(list.to_vec(), expected.to_vec()); + assert_eq!(list.tree_hash_root(), expected.tree_hash_root()); + } + } +} + +#[test] +fn replace_and_append_in_one_batch() { + // Mix replaces (existing indices) and appends (new indices) in a single `apply_updates`. + let mut list = build(10); + + *list.get_mut(0).unwrap() = 100; + *list.get_mut(9).unwrap() = 109; + list.push(10).unwrap(); + list.push(11).unwrap(); + + assert_eq!(list.len(), 12); + list.apply_updates().unwrap(); + + let mut expected_vec: Vec = (0..10).collect(); + expected_vec[0] = 100; + expected_vec[9] = 109; + expected_vec.push(10); + expected_vec.push(11); + let expected = ProgressiveList::::new(expected_vec).unwrap(); + + assert_eq!(list.to_vec(), expected.to_vec()); + assert_eq!(list.tree_hash_root(), expected.tree_hash_root()); +} + +#[test] +fn get_cow_and_iter_cow() { + let mut list = build(5); + + { + let c = list.get_cow(2).unwrap(); + assert_eq!(*c, 2); + *c.into_mut().unwrap() = 22; + } + assert_eq!(list.get(2).copied(), Some(22)); + + { + let mut iter = list.iter_cow(); + while let Some((index, v)) = iter.next_cow() { + *v.into_mut().unwrap() = (index * 10) as u64; + } + } + list.apply_updates().unwrap(); + assert_eq!(list.to_vec(), vec![0, 10, 20, 30, 40]); +} + +#[test] +fn iter_from_offset() { + for &n in TEST_LENGTHS { + let list = build(n); + for &k in &[0usize, n / 2, n] { + if k > n { + continue; + } + let collected: Vec = list.iter_from(k).unwrap().copied().collect(); + let expected: Vec = (k as u64..n as u64).collect(); + assert_eq!(collected, expected, "n={n}, k={k}"); + } + } +} + +#[test] +fn iter_from_with_pending_appends() { + let mut list = build(4); + list.push(4).unwrap(); + list.push(5).unwrap(); + + // Iterate including pending appends, starting partway through. + let collected: Vec = list.iter_from(2).unwrap().copied().collect(); + assert_eq!(collected, vec![2, 3, 4, 5]); +} + +#[test] +fn pop_front_matches_fresh() { + for &n in TEST_LENGTHS { + for &k in &[0usize, 1, n / 2, n] { + if k > n { + continue; + } + let mut list = build(n); + list.pop_front(k).unwrap(); + + let expected = ProgressiveList::::new((k as u64..n as u64).collect()).unwrap(); + assert_eq!(list.to_vec(), expected.to_vec(), "n={n}, k={k}"); + assert_eq!( + list.tree_hash_root(), + expected.tree_hash_root(), + "n={n}, k={k}" + ); + } + } +} + +#[test] +fn rebase_preserves_contents_and_root() { + for &n in TEST_LENGTHS { + if n == 0 { + continue; + } + // `orig` differs from `base` only in the last element, so earlier subtrees are equal. + let base = build(n); + + let mut orig_vec: Vec = (0..n as u64).collect(); + *orig_vec.last_mut().unwrap() = 9999; + let mut orig = ProgressiveList::::new(orig_vec.clone()).unwrap(); + + let root_before = orig.tree_hash_root(); + orig.rebase_on(&base).unwrap(); + + // Rebase must not change the logical value or the tree hash. + assert_eq!(orig.to_vec(), orig_vec, "n={n}"); + assert_eq!(orig.tree_hash_root(), root_before, "n={n}"); + } +} + +#[test] +fn rebase_shares_equal_subtrees() { + // Two independently-built equal lists have distinct backing trees until rebased. + let v: Vec = (0..20).collect(); + let base = ProgressiveList::::new(v.clone()).unwrap(); + let mut other = ProgressiveList::::new(v).unwrap(); + + let base_left = first_left(&base.tree).unwrap(); + let other_left_before = first_left(&other.tree).unwrap(); + assert!( + !Arc::ptr_eq(base_left, other_left_before), + "independently built lists should not share memory" + ); + + other.rebase_on(&base).unwrap(); + + let other_left_after = first_left(&other.tree).unwrap(); + assert!( + Arc::ptr_eq(base_left, other_left_after), + "equal left subtree should be shared after rebase" + ); +} + +#[test] +fn rebase_unequal_lists_keeps_correct_values() { + // Rebasing onto a shorter/different base must still yield the original contents. + let base = ProgressiveList::::new((0..5).collect()).unwrap(); + let orig_vec: Vec = (100..130).collect(); + let mut orig = ProgressiveList::::new(orig_vec.clone()).unwrap(); + + let root_before = orig.tree_hash_root(); + orig.rebase_on(&base).unwrap(); + + assert_eq!(orig.to_vec(), orig_vec); + assert_eq!(orig.tree_hash_root(), root_before); +} + +#[test] +fn rebase_with_pending_updates_applies_first() { + let base = build(10); + let mut orig = build(10); + + *orig.get_mut(3).unwrap() = 333; + assert!(orig.has_pending_updates()); + + orig.rebase_on(&base).unwrap(); + assert!(!orig.has_pending_updates()); + + let mut expected_vec: Vec = (0..10).collect(); + expected_vec[3] = 333; + let expected = ProgressiveList::::new(expected_vec).unwrap(); + assert_eq!(orig.to_vec(), expected.to_vec()); + assert_eq!(orig.tree_hash_root(), expected.tree_hash_root()); +} + +#[test] +fn mutation_matches_fresh_hash256() { + // Exercise an unpacked element type (no `PackedLeaf`). + let n = 40usize; + let mk = |i: usize| Hash256::repeat_byte(i as u8); + + let mut list = ProgressiveList::::new((0..n).map(mk).collect()).unwrap(); + *list.get_mut(37).unwrap() = Hash256::repeat_byte(0xff); + list.apply_updates().unwrap(); + + let mut expected_vec: Vec = (0..n).map(mk).collect(); + expected_vec[37] = Hash256::repeat_byte(0xff); + let expected = ProgressiveList::::new(expected_vec).unwrap(); + + assert_eq!(list.to_vec(), expected.to_vec()); + assert_eq!(list.tree_hash_root(), expected.tree_hash_root()); +} + +#[test] +fn bulk_update_then_apply() { + use crate::update_map::MaxMap; + use vec_map::VecMap; + + let mut list = build(8); + + let mut updates: MaxMap> = MaxMap::default(); + crate::UpdateMap::insert(&mut updates, 2, 222); + crate::UpdateMap::insert(&mut updates, 8, 888); // append + + list.bulk_update(updates).unwrap(); + assert_eq!(list.len(), 9); + list.apply_updates().unwrap(); + + let mut expected_vec: Vec = (0..8).collect(); + expected_vec[2] = 222; + expected_vec.push(888); + let expected = ProgressiveList::::new(expected_vec).unwrap(); + assert_eq!(list.to_vec(), expected.to_vec()); + assert_eq!(list.tree_hash_root(), expected.tree_hash_root()); +} diff --git a/src/tests/proptest/mod.rs b/src/tests/proptest/mod.rs index eaa9385..46d1116 100644 --- a/src/tests/proptest/mod.rs +++ b/src/tests/proptest/mod.rs @@ -35,6 +35,17 @@ pub fn arb_hash256() -> impl Strategy { proptest::array::uniform32(any::()).prop_map(Hash256::from) } +/// Strategy for generating initial values for a progressive list. +/// Unlike `arb_list`, this has no length limit, but we cap it at a reasonable +/// size for testing purposes. +pub fn arb_progressive_list(strategy: S, max_len: usize) -> impl Strategy> +where + S: Strategy, + T: std::fmt::Debug, +{ + proptest::collection::vec(strategy, 0..=max_len) +} + /// Struct with multiple fields shared by multiple proptests. #[derive(Debug, Clone, PartialEq, Encode, Decode, TreeHash)] pub struct Large { diff --git a/src/tests/proptest/operations.rs b/src/tests/proptest/operations.rs index 699a2fa..981acef 100644 --- a/src/tests/proptest/operations.rs +++ b/src/tests/proptest/operations.rs @@ -1,5 +1,5 @@ -use super::{Large, arb_hash256, arb_index, arb_large, arb_list, arb_vect}; -use crate::{Error, List, Value, Vector}; +use super::{Large, arb_hash256, arb_index, arb_large, arb_list, arb_progressive_list, arb_vect}; +use crate::{Error, List, ProgressiveList, Value, Vector}; use proptest::prelude::*; use ssz::{Decode, Encode}; use std::fmt::Debug; @@ -92,6 +92,62 @@ impl Spec { } } +/// Simple specification for `ProgressiveList` behaviour without a length limit. +#[derive(Debug, Clone)] +pub struct ProgressiveSpec { + values: Vec, +} + +impl ProgressiveSpec { + pub fn new(values: Vec) -> Self { + Self { values } + } + + pub fn len(&self) -> usize { + self.values.len() + } + + pub fn get(&self, index: usize) -> Option<&T> { + self.values.get(index) + } + + pub fn iter(&self) -> impl Iterator { + self.values.iter() + } + + pub fn push(&mut self, value: T) { + self.values.push(value); + } + + pub fn set(&mut self, index: usize, value: T) -> Option<()> { + *self.values.get_mut(index)? = value; + Some(()) + } + + pub fn iter_from(&self, index: usize) -> Result, Error> { + if index <= self.len() { + Ok(self.values[index..].iter()) + } else { + Err(Error::OutOfBoundsIterFrom { + index, + len: self.len(), + }) + } + } + + pub fn pop_front(&mut self, index: usize) -> Result<(), Error> { + if index <= self.len() { + self.values = self.values[index..].to_vec(); + Ok(()) + } else { + Err(Error::OutOfBoundsIterFrom { + index, + len: self.len(), + }) + } + } +} + #[derive(Debug, Clone)] pub enum Op { /// Check that `len` returns the correct length. @@ -177,6 +233,158 @@ where proptest::collection::vec(arb_op(strategy, n), 1..limit) } +/// Strategy for generating operations for `ProgressiveList`. +/// +/// Covers the full mutation/rebase API. `IntraRebase` and `FromIntoRoundtrip` are not applicable to +/// an (unbounded) `ProgressiveList` and are excluded. +fn arb_op_progressive<'a, T, S>( + strategy: &'a S, + max_len: usize, +) -> impl Strategy> + 'a +where + T: Debug + Clone + 'a, + S: Strategy + 'a, +{ + // The behaviour of `prop_oneof` changes to dynamic dispatch past 10 elements which + // breaks the borrowing pattern used in this function. Using two weighted substrategies + // prevents the boxing. + let a_block = prop_oneof![ + Just(Op::Len), + arb_index(max_len).prop_map(Op::Get), + (arb_index(max_len), strategy).prop_map(|(index, value)| Op::Set(index, value)), + (arb_index(max_len), strategy) + .prop_map(|(index, value)| Op::SetCowWithIntoMut(index, value)), + (arb_index(max_len), strategy) + .prop_map(|(index, value)| Op::SetCowWithMakeMut(index, value)), + strategy.prop_map(Op::Push), + Just(Op::Iter), + arb_index(max_len).prop_map(Op::IterFrom), + arb_index(max_len).prop_map(Op::IterCowFrom), + arb_index(max_len).prop_map(Op::PopFront), + ]; + let b_block = prop_oneof![ + Just(Op::ApplyUpdates), + Just(Op::TreeHash), + Just(Op::Checkpoint), + Just(Op::Rebase), + Just(Op::Debase), + ]; + prop_oneof![ + 10 => a_block, + 5 => b_block + ] +} + +fn arb_ops_progressive<'a, T, S>( + strategy: &'a S, + limit: usize, + max_len: usize, +) -> impl Strategy>> + 'a +where + T: Debug + Clone + 'a, + S: Strategy + 'a, +{ + proptest::collection::vec(arb_op_progressive(strategy, max_len), 1..limit) +} + +fn apply_ops_progressive_list( + list: &mut ProgressiveList, + spec: &mut ProgressiveSpec, + ops: Vec>, +) where + T: Value + Debug + Send + Sync, +{ + let mut checkpoint = list.clone(); + + for op in ops { + match op { + Op::Len => { + assert_eq!(list.len(), spec.len()); + } + Op::Get(index) => { + assert_eq!(list.get(index), spec.get(index)); + } + Op::Set(index, value) => { + let res = list.get_mut(index).map(|elem| *elem = value.clone()); + assert_eq!(res, spec.set(index, value)); + } + Op::SetCowWithIntoMut(index, value) => { + let res = list + .get_cow(index) + .map(|cow| *cow.into_mut().unwrap() = value.clone()); + assert_eq!(res, spec.set(index, value)); + } + Op::SetCowWithMakeMut(index, value) => { + let res = list + .get_cow(index) + .map(|mut cow| *cow.make_mut().unwrap() = value.clone()); + assert_eq!(res, spec.set(index, value)); + } + Op::Push(value) => { + list.push(value.clone()).expect("push should succeed"); + spec.push(value); + } + Op::Iter => { + assert!(list.iter().eq(spec.iter())); + } + Op::IterFrom(index) => match (list.iter_from(index), spec.iter_from(index)) { + (Ok(iter1), Ok(iter2)) => assert!(iter1.eq(iter2)), + (Err(e1), Err(e2)) => assert_eq!(e1, e2), + (Err(e), _) | (_, Err(e)) => panic!("iter_from mismatch: {}", e), + }, + Op::IterCowFrom(index) => match (list.iter_cow_from(index), spec.iter_from(index)) { + (Ok(mut cow_iter), Ok(spec_iter)) => { + let mut cow_values = Vec::new(); + while let Some((idx, cow)) = cow_iter.next_cow() { + assert_eq!( + idx, + index + cow_values.len(), + "index mismatch in iter_cow_from" + ); + cow_values.push(cow.deref().clone()); + } + assert!(cow_values.iter().eq(spec_iter)); + } + (Err(e1), Err(e2)) => assert_eq!(e1, e2), + (Err(e), _) | (_, Err(e)) => panic!("iter_cow_from mismatch: {}", e), + }, + Op::PopFront(index) => match (list.pop_front(index), spec.pop_front(index)) { + (Ok(()), Ok(())) => { + assert_eq!(list.len(), spec.len()); + assert!(list.iter().eq(spec.iter())) + } + (Err(e1), Err(e2)) => assert_eq!(e1, e2), + (Err(e), _) | (_, Err(e)) => panic!("pop_front mismatch: {}", e), + }, + Op::ApplyUpdates => { + list.apply_updates().unwrap(); + } + Op::TreeHash => { + list.apply_updates().unwrap(); + list.tree_hash_root(); + } + Op::Checkpoint => { + list.apply_updates().unwrap(); + checkpoint = list.clone(); + } + Op::Rebase => { + list.apply_updates().unwrap(); + let new_list = list.rebase(&checkpoint).unwrap(); + assert_eq!(new_list, *list); + } + Op::Debase => { + list.apply_updates().unwrap(); + let ssz_bytes = list.as_ssz_bytes(); + let new_list = ProgressiveList::from_ssz_bytes(&ssz_bytes).expect("SSZ decode"); + assert_eq!(new_list, *list); + *list = new_list; + } + // Not applicable to an (unbounded) `ProgressiveList`. + Op::FromIntoRoundtrip | Op::IntraRebase => {} + } + } +} + fn apply_ops_list(list: &mut List, spec: &mut Spec, ops: Vec>) where T: Value + Debug + Send + Sync, @@ -528,3 +736,37 @@ mod vect { vect_test!(large_33, Large, U33, arb_large()); vect_test!(large_1024, Large, U1024, arb_large()); } + +/// Maximum initial length for progressive list tests. +/// This is used to cap the initial list size for reasonable test execution time. +/// Compare to list tests which can use up to 1024 elements (U1024). +const PROGRESSIVE_LIST_MAX_LEN: usize = 128; + +macro_rules! progressive_list_test { + ($name:ident, $T:ty) => { + // Use default strategy (assumes existence of an `Arbitrary` impl). + progressive_list_test!($name, $T, any::<$T>()); + }; + ($name:ident, $T:ty, $strat:expr) => { + proptest! { + #[test] + fn $name( + init in arb_progressive_list::<$T, _>(&$strat, PROGRESSIVE_LIST_MAX_LEN), + ops in arb_ops_progressive::<$T, _>(&$strat, OP_LIMIT, PROGRESSIVE_LIST_MAX_LEN) + ) { + let mut list = ProgressiveList::<$T>::try_from_iter(init.clone()).unwrap(); + let mut spec = ProgressiveSpec::<$T>::new(init); + apply_ops_progressive_list(&mut list, &mut spec, ops); + } + } + }; +} + +mod progressive_list { + use super::*; + + progressive_list_test!(u8, u8); + progressive_list_test!(u64, u64); + progressive_list_test!(hash256, Hash256, arb_hash256()); + progressive_list_test!(large, Large, arb_large()); +}