From bbdb34f6ee062f9a1a3f6c10096b12e7ff97c932 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 1 Sep 2025 23:44:51 +0800 Subject: [PATCH 1/4] feat: cacheless iterator for hamt --- ipld/hamt/src/hamt.rs | 27 +++--- ipld/hamt/src/iter.rs | 193 ++++++++++++++++++++++++++++++++------- ipld/hamt/src/lib.rs | 2 +- ipld/hamt/src/node.rs | 14 +++ ipld/hamt/src/pointer.rs | 17 ++++ 5 files changed, 208 insertions(+), 45 deletions(-) diff --git a/ipld/hamt/src/hamt.rs b/ipld/hamt/src/hamt.rs index 73292c08ce..73cf02eaad 100644 --- a/ipld/hamt/src/hamt.rs +++ b/ipld/hamt/src/hamt.rs @@ -13,7 +13,7 @@ use multihash_codetable::Code; use serde::de::DeserializeOwned; use serde::{Serialize, Serializer}; -use crate::iter::IterImpl; +use crate::iter::{IterImpl, IterItem}; use crate::node::Node; use crate::pointer::version::Version; use crate::{Config, Error, Hash, HashAlgorithm, Sha256, pointer::version}; @@ -372,12 +372,13 @@ where #[inline] pub fn for_each(&self, mut f: F) -> Result<(), Error> where - V: DeserializeOwned, + K: Clone, + V: DeserializeOwned + Clone, F: FnMut(&K, &V) -> anyhow::Result<()>, { for res in self { let (k, v) = res?; - (f)(k, v)?; + (f)(k.as_ref(), v.as_ref())?; } Ok(()) } @@ -430,7 +431,7 @@ where where K: Borrow + Clone, Q: Eq + Hash + ?Sized, - V: DeserializeOwned, + V: DeserializeOwned + Clone, F: FnMut(&K, &V) -> anyhow::Result<()>, { let mut iter = match &starting_key { @@ -441,10 +442,10 @@ where let mut traversed = 0usize; for res in iter.by_ref().take(max.unwrap_or(usize::MAX)) { let (k, v) = res?; - (f)(k, v)?; + (f)(k.as_ref(), v.as_ref())?; traversed += 1; } - let next = iter.next().transpose()?.map(|kv| kv.0).cloned(); + let next = iter.next().transpose()?.map(|kv| kv.0.as_ref().clone()); Ok((traversed, next)) } @@ -456,8 +457,8 @@ where impl HamtImpl where - K: DeserializeOwned + PartialOrd, - V: DeserializeOwned, + K: DeserializeOwned + PartialOrd + Clone, + V: DeserializeOwned + Clone, Ver: Version, BS: Blockstore, { @@ -513,10 +514,10 @@ where /// for res in hamt.iter_from(results.last().unwrap().0)?.skip(1) { /// results.push((res?)); /// } - /// results.sort_by_key(|kv| kv.1); + /// results.sort_by_key(|kv| kv.1.clone()); /// /// // Assert that we got out what we put in. - /// let results: Vec<_> = results.into_iter().map(|(k, v)|(k.clone(), v.clone())).collect(); + /// let results: Vec<_> = results.into_iter().map(|(k, v)|(k.clone(), v.as_ref().clone())).collect(); /// assert_eq!(kvs, results); /// /// # anyhow::Ok(()) @@ -533,12 +534,12 @@ where impl<'a, BS, V, K, H, Ver> IntoIterator for &'a HamtImpl where - K: DeserializeOwned + PartialOrd, - V: DeserializeOwned, + K: DeserializeOwned + PartialOrd + Clone, + V: DeserializeOwned + Clone, Ver: Version, BS: Blockstore, { - type Item = Result<(&'a K, &'a V), Error>; + type Item = Result<(IterItem<'a, K>, IterItem<'a, V>), Error>; type IntoIter = IterImpl<'a, BS, V, K, H, Ver>; fn into_iter(self) -> Self::IntoIter { diff --git a/ipld/hamt/src/iter.rs b/ipld/hamt/src/iter.rs index dc2df6aede..6b48b317af 100644 --- a/ipld/hamt/src/iter.rs +++ b/ipld/hamt/src/iter.rs @@ -1,6 +1,7 @@ // Copyright 2021-2023 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT use std::borrow::Borrow; +use std::fmt::Display; use std::iter::FusedIterator; use forest_hash_utils::BytesKey; @@ -17,8 +18,8 @@ use crate::{Config, Error, Hash, HashAlgorithm, KeyValuePair, Sha256}; pub struct IterImpl<'a, BS, V, K = BytesKey, H = Sha256, Ver = version::V3> { store: &'a BS, conf: &'a Config, - stack: Vec>>, - current: std::slice::Iter<'a, KeyValuePair>, + stack: Vec>>, + current: StackItem<'a, KeyValuePair>, } /// Iterator over HAMT Key/Value tuples (hamt v0). @@ -27,10 +28,81 @@ pub type Iterv0<'a, BS, V, K = BytesKey, H = Sha256> = IterImpl<'a, BS, V, K, H, /// Iterator over HAMT Key/Value tuples. pub type Iter<'a, BS, V, K = BytesKey, H = Sha256> = IterImpl<'a, BS, V, K, H, version::V3>; +enum StackItem<'a, V> { + Iter(std::slice::Iter<'a, V>), + IntoIter(std::vec::IntoIter), +} + +impl<'a, V> From> for StackItem<'a, V> { + fn from(value: std::slice::Iter<'a, V>) -> Self { + Self::Iter(value) + } +} + +impl<'a, V> From> for StackItem<'a, V> { + fn from(value: std::vec::IntoIter) -> Self { + Self::IntoIter(value) + } +} + +impl<'a, V> Iterator for StackItem<'a, V> { + type Item = IterItem<'a, V>; + + fn next(&mut self) -> Option { + match self { + Self::Iter(it) => it.next().map(|i| i.into()), + Self::IntoIter(it) => it.next().map(|i| i.into()), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum IterItem<'a, V> { + Borrowed(&'a V), + Owned(V), +} + +impl Display for IterItem<'_, V> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_ref().fmt(f) + } +} + +impl AsRef for IterItem<'_, V> { + fn as_ref(&self) -> &V { + match self { + Self::Borrowed(v) => v, + Self::Owned(v) => v, + } + } +} + +impl PartialEq for IterItem<'_, V> { + fn eq(&self, other: &V) -> bool { + self.as_ref().eq(other) + } + + fn ne(&self, other: &V) -> bool { + self.as_ref().ne(other) + } +} + +impl<'a, V> From for IterItem<'a, V> { + fn from(value: V) -> Self { + Self::Owned(value) + } +} + +impl<'a, V> From<&'a V> for IterItem<'a, V> { + fn from(value: &'a V) -> Self { + Self::Borrowed(value) + } +} + impl<'a, K, V, BS, H, Ver> IterImpl<'a, BS, V, K, H, Ver> where - K: DeserializeOwned, - V: DeserializeOwned, + K: DeserializeOwned + Clone, + V: DeserializeOwned + Clone, Ver: Version, BS: Blockstore, { @@ -38,8 +110,8 @@ where Self { conf, store, - stack: vec![root.pointers.iter()], - current: [].iter(), + stack: vec![root.pointers.iter().into()], + current: [].iter().into(), } } @@ -56,24 +128,52 @@ where { let hashed_key = H::hash(key); let mut hash = HashBits::new(&hashed_key); - let mut node = root; + let mut node = IterItem::Borrowed(root); let mut stack = Vec::new(); loop { let idx = hash.next(conf.bit_width)?; - stack.push(node.pointers[node.index_for_bit_pos(idx)..].iter()); + match node.clone() { + IterItem::Borrowed(node) => { + stack.push(StackItem::from( + node.pointers[node.index_for_bit_pos(idx)..].iter(), + )); + } + IterItem::Owned(node) => { + stack.push(StackItem::from( + node.pointers[node.index_for_bit_pos(idx)..] + .to_vec() + .into_iter(), + )); + } + } node = match stack.last_mut().unwrap().next() { Some(p) => match p { - Pointer::Link { cid, cache } => cache.get_or_try_init(|| { - Node::load(conf, store, cid, stack.len() as u32).map(Box::new) - })?, - Pointer::Dirty(node) => node, - Pointer::Values(values) => { + IterItem::Borrowed(Pointer::Link { cid, cache: _ }) => { + Node::load(conf, store, cid, stack.len() as u32)?.into() + } + IterItem::Owned(Pointer::Link { cid, cache: _ }) => { + Node::load(conf, store, &cid, stack.len() as u32)?.into() + } + IterItem::Borrowed(Pointer::Dirty(node)) => node.as_ref().into(), + IterItem::Owned(Pointer::Dirty(node)) => (*node).into(), + IterItem::Borrowed(Pointer::Values(values)) => { + return match values.iter().position(|kv| kv.key().borrow() == key) { + Some(offset) => Ok(Self { + conf, + store, + stack, + current: values[offset..].iter().into(), + }), + None => Err(Error::StartKeyNotFound), + }; + } + IterItem::Owned(Pointer::Values(values)) => { return match values.iter().position(|kv| kv.key().borrow() == key) { Some(offset) => Ok(Self { conf, store, stack, - current: values[offset..].iter(), + current: values[offset..].to_vec().into_iter().into(), }), None => Err(Error::StartKeyNotFound), }; @@ -92,11 +192,13 @@ where K: DeserializeOwned + PartialOrd, V: DeserializeOwned, { - type Item = Result<(&'a K, &'a V), Error>; + type Item = Result<(IterItem<'a, K>, IterItem<'a, V>), Error>; fn next(&mut self) -> Option { - if let Some(v) = self.current.next() { - return Some(Ok((v.key(), v.value()))); + match self.current.next() { + Some(IterItem::Borrowed(v)) => return Some(Ok((v.key().into(), v.value().into()))), + Some(IterItem::Owned(KeyValuePair(k, v))) => return Some(Ok((k.into(), v.into()))), + _ => {} } loop { let Some(next) = self.stack.last_mut()?.next() else { @@ -104,21 +206,50 @@ where continue; }; match next { - Pointer::Link { cid, cache } => { - let node = match cache.get_or_try_init(|| { - Node::load(self.conf, &self.store, cid, self.stack.len() as u32) - .map(Box::new) - }) { - Ok(node) => node, - Err(e) => return Some(Err(e)), - }; - self.stack.push(node.pointers.iter()) + IterItem::Borrowed(Pointer::Link { cid, cache: _ }) => { + let node = + match Node::load(self.conf, &self.store, cid, self.stack.len() as u32) { + Ok(node) => node, + Err(e) => return Some(Err(e)), + }; + self.stack.push(node.pointers.into_iter().into()) + } + IterItem::Owned(Pointer::Link { cid, cache: _ }) => { + let node = + match Node::load(self.conf, &self.store, &cid, self.stack.len() as u32) { + Ok(node) => node, + Err(e) => return Some(Err(e)), + }; + self.stack.push(node.pointers.into_iter().into()) + } + IterItem::Borrowed(Pointer::Dirty(node)) => { + self.stack.push(node.pointers.iter().into()) + } + IterItem::Owned(Pointer::Dirty(node)) => { + self.stack.push(node.pointers.into_iter().into()) + } + IterItem::Borrowed(Pointer::Values(kvs)) => { + self.current = kvs.iter().into(); + match self.current.next() { + Some(IterItem::Borrowed(v)) => { + return Some(Ok((v.key().into(), v.value().into()))); + } + Some(IterItem::Owned(KeyValuePair(k, v))) => { + return Some(Ok((k.into(), v.into()))); + } + _ => {} + } } - Pointer::Dirty(node) => self.stack.push(node.pointers.iter()), - Pointer::Values(kvs) => { - self.current = kvs.iter(); - if let Some(v) = self.current.next() { - return Some(Ok((v.key(), v.value()))); + IterItem::Owned(Pointer::Values(kvs)) => { + self.current = kvs.into_iter().into(); + match self.current.next() { + Some(IterItem::Borrowed(v)) => { + return Some(Ok((v.key().into(), v.value().into()))); + } + Some(IterItem::Owned(KeyValuePair(k, v))) => { + return Some(Ok((k.into(), v.into()))); + } + _ => {} } } } diff --git a/ipld/hamt/src/lib.rs b/ipld/hamt/src/lib.rs index b8425d551f..17d47c888d 100644 --- a/ipld/hamt/src/lib.rs +++ b/ipld/hamt/src/lib.rs @@ -75,7 +75,7 @@ impl Default for Config { type HashedKey = [u8; 32]; -#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct KeyValuePair(K, V); impl KeyValuePair { diff --git a/ipld/hamt/src/node.rs b/ipld/hamt/src/node.rs index 6c7a34090f..da527e41f1 100644 --- a/ipld/hamt/src/node.rs +++ b/ipld/hamt/src/node.rs @@ -29,6 +29,20 @@ pub(crate) struct Node { hash: PhantomData, } +impl Clone for Node +where + K: Clone, + V: Clone, +{ + fn clone(&self) -> Self { + Self { + bitfield: self.bitfield.clone(), + pointers: self.pointers.clone(), + hash: Default::default(), + } + } +} + impl PartialEq for Node { fn eq(&self, other: &Self) -> bool { (self.bitfield == other.bitfield) && (self.pointers == other.pointers) diff --git a/ipld/hamt/src/pointer.rs b/ipld/hamt/src/pointer.rs index b33e6c5834..fd6e4d6d01 100644 --- a/ipld/hamt/src/pointer.rs +++ b/ipld/hamt/src/pointer.rs @@ -46,6 +46,23 @@ pub(crate) enum Pointer { Dirty(Box>), } +impl Clone for Pointer +where + K: Clone, + V: Clone, +{ + fn clone(&self) -> Self { + match self { + Self::Values(v) => Self::Values(v.clone()), + Self::Link { cid, cache: _ } => Self::Link { + cid: *cid, + cache: Default::default(), + }, + Self::Dirty(n) => Self::Dirty(n.clone()), + } + } +} + impl PartialEq for Pointer { fn eq(&self, other: &Self) -> bool { match (self, other) { From 389a74d759a1085e5829e7ed8287a099831cb99d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 2 Sep 2025 10:52:16 +0800 Subject: [PATCH 2/4] update tests --- Cargo.lock | 1 + ipld/hamt/Cargo.toml | 1 + ipld/hamt/src/hamt.rs | 4 +- ipld/hamt/tests/hamt_tests.rs | 145 ++++++++++++++++++++-------------- 4 files changed, 89 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7783e3466a..f9cb43d9b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2527,6 +2527,7 @@ dependencies = [ "fvm_ipld_encoding 0.5.3", "hex", "ipld-core", + "itertools 0.14.0", "multihash-codetable", "once_cell", "quickcheck", diff --git a/ipld/hamt/Cargo.toml b/ipld/hamt/Cargo.toml index a8a32f8b50..ee018e2f18 100644 --- a/ipld/hamt/Cargo.toml +++ b/ipld/hamt/Cargo.toml @@ -28,6 +28,7 @@ identity = [] [dev-dependencies] hex = { workspace = true } criterion = { workspace = true } +itertools = { workspace = true} unsigned-varint = { workspace = true } quickcheck = { workspace = true } quickcheck_macros = { workspace = true } diff --git a/ipld/hamt/src/hamt.rs b/ipld/hamt/src/hamt.rs index 73cf02eaad..1004436180 100644 --- a/ipld/hamt/src/hamt.rs +++ b/ipld/hamt/src/hamt.rs @@ -511,13 +511,13 @@ where /// assert_eq!(results.len(), 2); /// /// // Read the rest then sort. - /// for res in hamt.iter_from(results.last().unwrap().0)?.skip(1) { + /// for res in hamt.iter_from(results.last().unwrap().0.as_ref())?.skip(1) { /// results.push((res?)); /// } /// results.sort_by_key(|kv| kv.1.clone()); /// /// // Assert that we got out what we put in. - /// let results: Vec<_> = results.into_iter().map(|(k, v)|(k.clone(), v.as_ref().clone())).collect(); + /// let results: Vec<_> = results.into_iter().map(|(k, v)|(k.as_ref().clone(), v.as_ref().clone())).collect(); /// assert_eq!(kvs, results); /// /// # anyhow::Ok(()) diff --git a/ipld/hamt/tests/hamt_tests.rs b/ipld/hamt/tests/hamt_tests.rs index a761502d4b..5c894364bf 100644 --- a/ipld/hamt/tests/hamt_tests.rs +++ b/ipld/hamt/tests/hamt_tests.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT use std::collections::{HashMap, HashSet}; -use std::fmt::Display; +use std::fmt::{Debug, Display}; use cid::Cid; use fvm_ipld_blockstore::tracking::{BSStats, TrackingBlockstore}; @@ -14,6 +14,7 @@ use fvm_ipld_encoding::strict_bytes::ByteBuf; #[cfg(feature = "identity")] use fvm_ipld_hamt::Identity; use fvm_ipld_hamt::{BytesKey, Config, Error, Hamt, Hash}; +use itertools::Itertools as _; use multihash_codetable::Code; use quickcheck::Arbitrary; use rand::SeedableRng; @@ -84,44 +85,61 @@ impl HamtFactory { } } -/// Check hard-coded CIDs during testing. -struct CidChecker { +type CidChecker = Checker; + +type BSStatsChecker = Checker; + +/// Check hard-coded values during testing. +struct Checker { checked: usize, - cids: Option>, + expected_values: Option>, } -impl CidChecker { - pub fn new(cids: Vec<&'static str>) -> Self { +impl Checker +where + T: Debug + PartialEq, +{ + pub fn new(expected_values: impl IntoIterator) -> Self + where + T: TryFrom, + >::Error: Debug, + { Self { - cids: Some(cids), + expected_values: Some( + expected_values + .into_iter() + .map(T::try_from) + .try_collect() + .unwrap(), + ), checked: 0, } } pub fn empty() -> Self { Self { - cids: None, + expected_values: None, checked: 0, } } - pub fn check_next(&mut self, cid: Cid) { - if let Some(cids) = &self.cids { - assert_ne!(self.checked, cids.len()); - assert_eq!(cid.to_string().as_str(), cids[self.checked]); + pub fn check_next(&mut self, actual_value: &T) { + if let Some(expected_values) = &self.expected_values { + assert_ne!(self.checked, expected_values.len()); + assert_eq!(actual_value, &expected_values[self.checked]); self.checked += 1; } } } -impl Drop for CidChecker { +impl Drop for Checker { fn drop(&mut self) { if std::thread::panicking() { // Already failed, don't double-panic. return; } - if let Some(cids) = &self.cids { - assert_eq!(self.checked, cids.len()) + if let Some(expected_values) = &self.expected_values { + assert_eq!(self.checked, expected_values.len()) } } } @@ -302,7 +320,7 @@ fn test_set_if_absent(factory: HamtFactory, stats: Option, mut cids: Ci .unwrap() ); - cids.check_next(c); + cids.check_next(&c); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); @@ -324,12 +342,12 @@ fn set_with_no_effect_does_not_put( } let c = begn.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); begn.set(tstring("favorite-animal"), tstring("bright green bear")) .unwrap(); let c2 = begn.flush().unwrap(); - cids.check_next(c2); + cids.check_next(&c2); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); } @@ -337,7 +355,7 @@ fn set_with_no_effect_does_not_put( begn.set(tstring("favorite-animal"), tstring("bright green bear")) .unwrap(); let c3 = begn.flush().unwrap(); - cids.check_next(c3); + cids.check_next(&c3); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); @@ -356,7 +374,7 @@ fn delete(factory: HamtFactory, stats: Option, mut cids: CidChecker) { assert!(hamt.contains_key(&tstring("foo")).unwrap()); let c = hamt.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); let mut h2: Hamt<_, BytesKey> = factory.load(&c, &store).unwrap(); assert!(h2.get(&b"foo".to_vec()).unwrap().is_some()); @@ -368,7 +386,7 @@ fn delete(factory: HamtFactory, stats: Option, mut cids: CidChecker) { assert!(h2.delete(&b"nonexistent".to_vec()).unwrap().is_none()); let c2 = h2.flush().unwrap(); - cids.check_next(c2); + cids.check_next(&c2); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); } @@ -384,14 +402,14 @@ fn delete_case(factory: HamtFactory, stats: Option, mut cids: CidChecke .unwrap(); let c = hamt.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); let mut h2: Hamt<_, ByteBuf> = factory.load(&c, &store).unwrap(); assert!(h2.delete(&[0].to_vec()).unwrap().is_some()); assert_eq!(h2.get(&[0].to_vec()).unwrap(), None); let c2 = h2.flush().unwrap(); - cids.check_next(c2); + cids.check_next(&c2); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); } @@ -407,7 +425,7 @@ fn reload_empty(factory: HamtFactory, stats: Option, mut cids: CidCheck let h2: Hamt<_, ()> = factory.load(&c, &store).unwrap(); let c2 = store.put_cbor(&h2, Code::Blake2b256).unwrap(); assert_eq!(c, c2); - cids.check_next(c); + cids.check_next(&c); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); } @@ -430,14 +448,14 @@ fn set_delete_many( } let c1 = hamt.flush().unwrap(); - cids.check_next(c1); + cids.check_next(&c1); for i in size_factor..(size_factor * 2) { hamt.set(tstring(i), tstring(i)).unwrap(); } let cid_all = hamt.flush().unwrap(); - cids.check_next(cid_all); + cids.check_next(&cid_all); for i in size_factor..(size_factor * 2) { assert!(hamt.delete(&tstring(i)).unwrap().is_some()); @@ -449,7 +467,7 @@ fn set_delete_many( } let cid_d = hamt.flush().unwrap(); - cids.check_next(cid_d); + cids.check_next(&cid_d); // Assert that we can empty it. for i in 0..size_factor { @@ -460,7 +478,7 @@ fn set_delete_many( assert_eq!(hamt.iter().count(), 0); let cid_d = hamt.flush().unwrap(); - cids.check_next(cid_d); + cids.check_next(&cid_d); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); } @@ -473,7 +491,7 @@ fn set_delete_many( fn for_each( size_factor: usize, factory: HamtFactory, - stats: Option, + mut stats: Option, mut cids: CidChecker, ) { let mem = MemoryBlockstore::default(); @@ -496,7 +514,10 @@ fn for_each( assert_eq!(count, size_factor); let c = hamt.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); + if let Some(stats) = &mut stats { + stats.check_next(&*store.stats.borrow()); + } let mut hamt: Hamt<_, BytesKey> = factory.load_with_bit_width(&c, &store, 5).unwrap(); @@ -510,19 +531,10 @@ fn for_each( .unwrap(); assert_eq!(count, size_factor); - // Iterating through hamt with cached nodes. - let mut count = 0; - hamt.for_each(|k, v| { - assert_eq!(k, v); - count += 1; - Ok(()) - }) - .unwrap(); - assert_eq!(count, size_factor); - - { - let c = hamt.flush().unwrap(); - cids.check_next(c); + let c = hamt.flush().unwrap(); + cids.check_next(&c); + if let Some(stats) = &mut stats { + stats.check_next(&*store.stats.borrow()); } // Iterate with a few modified nodes. @@ -573,17 +585,16 @@ fn for_each( assert_eq!(count, expected_count); let c = hamt.flush().unwrap(); - cids.check_next(c); - - if let Some(stats) = stats { - assert_eq!(*store.stats.borrow(), stats); + cids.check_next(&c); + if let Some(stats) = &mut stats { + stats.check_next(&*store.stats.borrow()); } } fn for_each_ranged( size_factor: usize, factory: HamtFactory, - stats: Option, + mut stats: Option, mut cids: CidChecker, ) { let mem = MemoryBlockstore::default(); @@ -698,7 +709,10 @@ fn for_each_ranged( } let c = hamt.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); + if let Some(stats) = &mut stats { + stats.check_next(&*store.stats.borrow()); + } // Chain paginated requests over a HAMT with committed nodes let mut hamt: Hamt<_, usize> = factory.load_with_bit_width(&c, &store, 5).unwrap(); @@ -749,7 +763,10 @@ fn for_each_ranged( } let c = hamt.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); + if let Some(stats) = &mut stats { + stats.check_next(&*store.stats.borrow()); + } // Test modifications and deletions in ranged iteration if size_factor > 10 { @@ -782,10 +799,9 @@ fn for_each_ranged( assert_eq!(kvs_after_mod.len(), expected_count); let c = hamt.flush().unwrap(); - cids.check_next(c); - - if let Some(stats) = stats { - assert_eq!(*store.stats.borrow(), stats); + cids.check_next(&c); + if let Some(stats) = &mut stats { + stats.check_next(&*store.stats.borrow()); } } @@ -828,7 +844,7 @@ fn clear(factory: HamtFactory, mut cids: CidChecker) { assert_eq!(hamt.get(&3).unwrap(), Some(&"c".to_string())); let c = hamt.flush().unwrap(); - cids.check_next(c); + cids.check_next(&c); } #[cfg(feature = "identity")] @@ -973,7 +989,7 @@ fn clean_child_ordering(factory: HamtFactory, stats: Option, mut cids: } let root = h.flush().unwrap(); - cids.check_next(root); + cids.check_next(&root); let mut h: Hamt<_, u8> = factory.load_with_bit_width(&root, &store, 5).unwrap(); h.delete(&make_key(104)).unwrap(); @@ -981,7 +997,7 @@ fn clean_child_ordering(factory: HamtFactory, stats: Option, mut cids: let root = h.flush().unwrap(); let _: Hamt<_, u8> = factory.load_with_bit_width(&root, &store, 5).unwrap(); - cids.check_next(root); + cids.check_next(&root); if let Some(stats) = stats { assert_eq!(*store.stats.borrow(), stats); @@ -1181,7 +1197,7 @@ mod test_default { use fvm_ipld_hamt::{Config, Hamtv0}; use quickcheck_macros::quickcheck; - use crate::{CidChecker, HamtFactory, LimitedKeyOps, UniqueKeyValuePairs}; + use crate::{BSStatsChecker, CidChecker, HamtFactory, LimitedKeyOps, UniqueKeyValuePairs}; #[test] fn test_basics() { @@ -1273,7 +1289,11 @@ mod test_default { #[test] fn for_each() { #[rustfmt::skip] - let stats = BSStats {r: 30, w: 33, br: 3209, bw: 4697}; + let stats = BSStatsChecker::new(vec![ + BSStats {r: 0, w: 30, br: 0, bw: 3209}, + BSStats {r: 30, w: 30, br: 3209, bw: 3209}, + BSStats {r: 60, w: 33, br: 5158, bw: 4697}, + ]); let cids = CidChecker::new(vec![ "bafy2bzaceczhz54xmmz3xqnbmvxfbaty3qprr6dq7xh5vzwqbirlsnbd36z7a", "bafy2bzaceczhz54xmmz3xqnbmvxfbaty3qprr6dq7xh5vzwqbirlsnbd36z7a", @@ -1285,7 +1305,12 @@ mod test_default { #[test] fn for_each_ranged() { #[rustfmt::skip] - let stats = BSStats {r: 30, w: 33, br: 2895, bw: 4321}; + #[rustfmt::skip] + let stats = BSStatsChecker::new(vec![ + BSStats {r: 0, w: 30, br: 0, bw: 2895}, + BSStats {r: 39, w: 30, br: 3410, bw: 2895}, + BSStats {r: 68, w: 33, br: 4997, bw: 4321}, + ]); let cids = CidChecker::new(vec![ "bafy2bzacedy4ypl2vedhdqep3llnwko6vrtfiys5flciz2f3c55pl4whlhlqm", "bafy2bzacedy4ypl2vedhdqep3llnwko6vrtfiys5flciz2f3c55pl4whlhlqm", From dad03d36372fece630b9bf15bf995eca4473ceea Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 2 Sep 2025 11:15:52 +0800 Subject: [PATCH 3/4] make clippy happy --- ipld/hamt/src/hamt.rs | 2 +- ipld/hamt/src/iter.rs | 23 ++++------------------- ipld/hamt/src/node.rs | 2 +- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/ipld/hamt/src/hamt.rs b/ipld/hamt/src/hamt.rs index 1004436180..bceab33a95 100644 --- a/ipld/hamt/src/hamt.rs +++ b/ipld/hamt/src/hamt.rs @@ -477,7 +477,7 @@ where /// /// for kv in &hamt { /// let (k, v) = kv?; - /// println!("{k:?}: {v}"); + /// println!("{:?}: {}", k.as_ref(), v.as_ref()); /// } /// /// # anyhow::Ok(()) diff --git a/ipld/hamt/src/iter.rs b/ipld/hamt/src/iter.rs index 6b48b317af..7a3f8cb87d 100644 --- a/ipld/hamt/src/iter.rs +++ b/ipld/hamt/src/iter.rs @@ -1,7 +1,6 @@ // Copyright 2021-2023 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT use std::borrow::Borrow; -use std::fmt::Display; use std::iter::FusedIterator; use forest_hash_utils::BytesKey; @@ -39,7 +38,7 @@ impl<'a, V> From> for StackItem<'a, V> { } } -impl<'a, V> From> for StackItem<'a, V> { +impl From> for StackItem<'_, V> { fn from(value: std::vec::IntoIter) -> Self { Self::IntoIter(value) } @@ -62,12 +61,6 @@ pub enum IterItem<'a, V> { Owned(V), } -impl Display for IterItem<'_, V> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.as_ref().fmt(f) - } -} - impl AsRef for IterItem<'_, V> { fn as_ref(&self) -> &V { match self { @@ -77,17 +70,7 @@ impl AsRef for IterItem<'_, V> { } } -impl PartialEq for IterItem<'_, V> { - fn eq(&self, other: &V) -> bool { - self.as_ref().eq(other) - } - - fn ne(&self, other: &V) -> bool { - self.as_ref().ne(other) - } -} - -impl<'a, V> From for IterItem<'a, V> { +impl From for IterItem<'_, V> { fn from(value: V) -> Self { Self::Owned(value) } @@ -140,6 +123,7 @@ where } IterItem::Owned(node) => { stack.push(StackItem::from( + #[allow(clippy::unnecessary_to_owned)] node.pointers[node.index_for_bit_pos(idx)..] .to_vec() .into_iter(), @@ -173,6 +157,7 @@ where conf, store, stack, + #[allow(clippy::unnecessary_to_owned)] current: values[offset..].to_vec().into_iter().into(), }), None => Err(Error::StartKeyNotFound), diff --git a/ipld/hamt/src/node.rs b/ipld/hamt/src/node.rs index da527e41f1..cf5f90f1b3 100644 --- a/ipld/hamt/src/node.rs +++ b/ipld/hamt/src/node.rs @@ -36,7 +36,7 @@ where { fn clone(&self) -> Self { Self { - bitfield: self.bitfield.clone(), + bitfield: self.bitfield, pointers: self.pointers.clone(), hash: Default::default(), } From e5c9ea9dc2da5694eb32fc06f2066da03359007f Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 2 Sep 2025 11:32:18 +0800 Subject: [PATCH 4/4] versioni bump --- Cargo.lock | 52 ++++++++++++++++++++++---------------------- Cargo.toml | 2 +- ipld/hamt/Cargo.toml | 2 +- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9cb43d9b2..d58b98a29d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1375,7 +1375,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static", "log", @@ -1457,7 +1457,7 @@ dependencies = [ "frc42_dispatch", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "log", "num-derive", @@ -1478,7 +1478,7 @@ dependencies = [ "fvm_ipld_bitfield 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "integer-encoding", "ipld-core", @@ -1505,7 +1505,7 @@ dependencies = [ "fvm_ipld_bitfield 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.14.0", "lazy_static", @@ -1529,7 +1529,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 2.9.0", "integer-encoding", @@ -1571,7 +1571,7 @@ dependencies = [ "frc42_dispatch", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 2.9.0", "integer-encoding", @@ -1628,7 +1628,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static", "log", @@ -1663,7 +1663,7 @@ dependencies = [ "fvm_ipld_bitfield 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_sdk 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "integer-encoding", @@ -2091,7 +2091,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_sdk 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "integer-encoding", @@ -2222,7 +2222,7 @@ dependencies = [ "fvm_ipld_amt 0.7.5", "fvm_ipld_blockstore 0.3.1", "fvm_ipld_encoding 0.5.3", - "fvm_ipld_hamt 0.10.4", + "fvm_ipld_hamt 0.11.0", "fvm_shared 4.7.3", "lazy_static", "log", @@ -2517,46 +2517,46 @@ dependencies = [ [[package]] name = "fvm_ipld_hamt" version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e92fa6ad9ebdb821f7d3183666a94b6fabd6640d5c83ce1cd850865746d2e4db" dependencies = [ "anyhow", "byteorder", "cid", - "criterion", "forest_hash_utils", - "fvm_ipld_blockstore 0.3.1", - "fvm_ipld_encoding 0.5.3", - "hex", + "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "ipld-core", - "itertools 0.14.0", "multihash-codetable", "once_cell", - "quickcheck", - "quickcheck_macros", - "rand", "serde", "sha2", "thiserror 2.0.12", - "unsigned-varint", ] [[package]] name = "fvm_ipld_hamt" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e92fa6ad9ebdb821f7d3183666a94b6fabd6640d5c83ce1cd850865746d2e4db" +version = "0.11.0" dependencies = [ "anyhow", "byteorder", "cid", + "criterion", "forest_hash_utils", - "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_blockstore 0.3.1", + "fvm_ipld_encoding 0.5.3", + "hex", "ipld-core", + "itertools 0.14.0", "multihash-codetable", "once_cell", + "quickcheck", + "quickcheck_macros", + "rand", "serde", "sha2", "thiserror 2.0.12", + "unsigned-varint", ] [[package]] @@ -3082,7 +3082,7 @@ version = "0.0.0" dependencies = [ "arbitrary", "fvm_ipld_blockstore 0.3.1", - "fvm_ipld_hamt 0.10.4", + "fvm_ipld_hamt 0.11.0", "libfuzzer-sys", ] @@ -4878,7 +4878,7 @@ dependencies = [ "cid", "fvm_ipld_blockstore 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fvm_ipld_encoding 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fvm_ipld_hamt 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fvm_ipld_hamt 0.10.4", "fvm_shared 4.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "multihash-codetable", "num-derive", diff --git a/Cargo.toml b/Cargo.toml index 9a955f1570..aebfe145aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,7 +79,7 @@ fvm_integration_tests = { path = "testing/integration", version = "~4.7.3" } # workspace (other) fvm_ipld_amt = { path = "ipld/amt", version = "0.7.5" } -fvm_ipld_hamt = { path = "ipld/hamt", version = "0.10.4" } +fvm_ipld_hamt = { path = "ipld/hamt", version = "0.11.0" } fvm_ipld_kamt = { path = "ipld/kamt", version = "0.4.5" } fvm_ipld_car = { path = "ipld/car", version = "0.9.0" } fvm_ipld_blockstore = { path = "ipld/blockstore", version = "0.3.1" } diff --git a/ipld/hamt/Cargo.toml b/ipld/hamt/Cargo.toml index ee018e2f18..3e4d94e6f2 100644 --- a/ipld/hamt/Cargo.toml +++ b/ipld/hamt/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "fvm_ipld_hamt" description = "Sharded IPLD HashMap implementation." -version = "0.10.4" +version = "0.11.0" license.workspace = true authors = ["ChainSafe Systems ", "Protocol Labs", "Filecoin Core Devs"] edition.workspace = true