feat: refactor Merkle proof for ergonomics (#692)

* Remove index and element from Merkle proof. Temporarily remove NMT

* fmt & test_serde & deprecate NMT

* CHANGELOG

* address commments for `verify_merkle_proof`

* address comments

* change commitment to a single digest value

* fix gadgets

merkle_tree/CHANGELOG.md

@@ -3,6 +3,15 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.2.0 (2024-10-21)
+
+- [#692](https://github.com/EspressoSystems/jellyfish/pull/692) Major refactor for ergonomics reason
+    - `MerkleProof` now doesn't contain leaf information. Proofs should be verified along with claimed 
+      index and element information.
+    - Merkle proof verification proof APIs now takes `MerkleCommitment` instead of simply a root digest 
+      value. It can now be called without instantiating an actual Merkle tree struct.
+    - Deprecate namespace Merkle tree for now because it's no longer in use.
+
 ## 0.1.0
 
 - Initial release. 

merkle_tree/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jf-merkle-tree"
-version = "0.1.0"
+version = "0.2.0"
 description = "Various Merkle tree implementations."
 authors = { workspace = true }
 edition = { workspace = true }

merkle_tree/benches/merkle_path.rs

@@ -10,7 +10,7 @@ extern crate criterion;
 use ark_ed_on_bls12_381::Fq as Fq381;
 use ark_std::rand::Rng;
 use criterion::Criterion;
-use jf_merkle_tree::{prelude::RescueMerkleTree, MerkleCommitment, MerkleTreeScheme};
+use jf_merkle_tree::{prelude::RescueMerkleTree, MerkleTreeScheme};
 use std::time::Duration;
 
 const BENCH_NAME: &str = "merkle_path_height_20";
@@ -25,12 +25,12 @@ fn twenty_hashes(c: &mut Criterion) {
     let leaf: Fq381 = rng.gen();
 
     let mt = RescueMerkleTree::<Fq381>::from_elems(Some(20), [leaf, leaf]).unwrap();
-    let root = mt.commitment().digest();
-    let (_, proof) = mt.lookup(0).expect_ok().unwrap();
+    let commitment = mt.commitment();
+    let (val, proof) = mt.lookup(0).expect_ok().unwrap();
 
     let num_inputs = 0;
     benchmark_group.bench_with_input(BENCH_NAME, &num_inputs, move |b, &_num_inputs| {
-        b.iter(|| RescueMerkleTree::<Fq381>::verify(&root, 0, &proof).unwrap())
+        b.iter(|| RescueMerkleTree::<Fq381>::verify(&commitment, 0, val, &proof).unwrap())
     });
     benchmark_group.finish();
 }

merkle_tree/src/append_only.rs

@@ -8,11 +8,10 @@
 
 use super::{
     internal::{
-        build_tree_internal, MerkleNode, MerkleProof, MerkleTreeCommitment, MerkleTreeIntoIter,
-        MerkleTreeIter,
+        build_tree_internal, MerkleNode, MerkleTreeIntoIter, MerkleTreeIter, MerkleTreeProof,
     },
     AppendableMerkleTreeScheme, DigestAlgorithm, Element, ForgetableMerkleTreeScheme, Index,
-    LookupResult, MerkleCommitment, MerkleTreeScheme, NodeValue, ToTraversalPath,
+    LookupResult, MerkleProof, MerkleTreeScheme, NodeValue, ToTraversalPath,
 };
 use crate::{
     errors::MerkleTreeError, impl_forgetable_merkle_tree_scheme, impl_merkle_tree_scheme,
@@ -104,12 +103,10 @@ where
     }
 }
 
-// TODO(Chengyu): extract a merkle frontier
-
 #[cfg(test)]
 mod mt_tests {
     use crate::{
-        internal::{MerkleNode, MerkleProof},
+        internal::{MerkleNode, MerkleTreeProof},
         prelude::{RescueMerkleTree, RescueSparseMerkleTree},
         *,
     };
@@ -166,43 +163,36 @@ mod mt_tests {
 
         let mt =
             RescueMerkleTree::<F>::from_elems(Some(2), [F::from(3u64), F::from(1u64)]).unwrap();
-        let root = mt.commitment().digest();
+        let commitment = mt.commitment();
         let (elem, proof) = mt.lookup(0).expect_ok().unwrap();
         assert_eq!(elem, &F::from(3u64));
-        assert_eq!(proof.tree_height(), 3);
-        assert!(RescueMerkleTree::<F>::verify(&root, 0u64, &proof)
+        assert_eq!(proof.height(), 2);
+        assert!(
+            RescueMerkleTree::<F>::verify(&commitment, 0u64, elem, &proof)
+                .unwrap()
+                .is_ok()
+        );
+
+        // Wrong element value, should fail.
+        assert!(
+            RescueMerkleTree::<F>::verify(&commitment, 0, F::from(14u64), &proof)
+                .unwrap()
+                .is_err()
+        );
+
+        // Wrong pos, should fail.
+        assert!(RescueMerkleTree::<F>::verify(&commitment, 1, elem, &proof)
             .unwrap()
-            .is_ok());
+            .is_err());
 
         let mut bad_proof = proof.clone();
-        if let MerkleNode::Leaf {
-            value: _,
-            pos: _,
-            elem,
-        } = &mut bad_proof.proof[0]
-        {
-            *elem = F::from(4u64);
-        } else {
-            unreachable!()
-        }
+        bad_proof.0[0][0] = F::one();
 
-        let result = RescueMerkleTree::<F>::verify(&root, 0, &bad_proof);
-        assert!(result.unwrap().is_err());
-
-        let mut forge_proof = MerkleProof::new(2, proof.proof);
-        if let MerkleNode::Leaf {
-            value: _,
-            pos,
-            elem,
-        } = &mut forge_proof.proof[0]
-        {
-            *pos = 2;
-            *elem = F::from(0u64);
-        } else {
-            unreachable!()
-        }
-        let result = RescueMerkleTree::<F>::verify(&root, 0, &forge_proof);
-        assert!(result.unwrap().is_err());
+        assert!(
+            RescueMerkleTree::<F>::verify(&commitment, 0, elem, &bad_proof)
+                .unwrap()
+                .is_err()
+        );
     }
 
     #[test]
@@ -213,55 +203,67 @@ mod mt_tests {
     }
 
     fn test_mt_forget_remember_helper<F: RescueParameter>() {
-        let mut mt =
-            RescueMerkleTree::<F>::from_elems(Some(2), [F::from(3u64), F::from(1u64)]).unwrap();
-        let root = mt.commitment().digest();
-        let (lookup_elem, lookup_proof) = mt.lookup(0).expect_ok().unwrap();
-        let lookup_elem = *lookup_elem;
-        let (elem, proof) = mt.forget(0).expect_ok().unwrap();
+        let mut mt = RescueMerkleTree::<F>::from_elems(
+            Some(2),
+            [F::from(3u64), F::from(1u64), F::from(2u64), F::from(5u64)],
+        )
+        .unwrap();
+        let commitment = mt.commitment();
+        let (&lookup_elem, mut lookup_proof) = mt.lookup(3).expect_ok().unwrap();
+        let (elem, proof) = mt.forget(3).expect_ok().unwrap();
         assert_eq!(lookup_elem, elem);
         assert_eq!(lookup_proof, proof);
-        assert_eq!(elem, F::from(3u64));
-        assert_eq!(proof.tree_height(), 3);
-        assert!(RescueMerkleTree::<F>::verify(&root, 0, &lookup_proof)
+        assert_eq!(elem, F::from(5u64));
+        assert_eq!(proof.height(), 2);
+        assert!(
+            RescueMerkleTree::<F>::verify(&commitment, 3, elem, &lookup_proof)
+                .unwrap()
+                .is_ok()
+        );
+        assert!(RescueMerkleTree::<F>::verify(&commitment, 3, elem, &proof)
             .unwrap()
             .is_ok());
-        assert!(RescueMerkleTree::<F>::verify(&root, 0, &proof)
+
+        assert!(mt.forget(3).expect_ok().is_err());
+        assert!(matches!(mt.lookup(3), LookupResult::NotInMemory));
+
+        // Wrong element
+        assert!(mt.remember(3, F::from(19u64), &proof).is_err());
+        // Wrong pos
+        assert!(mt.remember(1, elem, &proof).is_err());
+        // Wrong proof
+        lookup_proof.0[0][0] = F::one();
+        assert!(mt.remember(3, elem, &lookup_proof).is_err());
+
+        assert!(mt.remember(3, elem, &proof).is_ok());
+        assert!(mt.lookup(3).expect_ok().is_ok());
+
+        // test another index
+        let (&lookup_elem, mut lookup_proof) = mt.lookup(0).expect_ok().unwrap();
+        let (elem, proof) = mt.forget(0).expect_ok().unwrap();
+        assert_eq!(lookup_elem, elem);
+        assert_eq!(lookup_proof, proof);
+        assert_eq!(elem, F::from(3u64));
+        assert_eq!(proof.height(), 2);
+        assert!(
+            RescueMerkleTree::<F>::verify(&commitment, 0, elem, &lookup_proof)
+                .unwrap()
+                .is_ok()
+        );
+        assert!(RescueMerkleTree::<F>::verify(&commitment, 0, elem, &proof)
             .unwrap()
             .is_ok());
 
         assert!(mt.forget(0).expect_ok().is_err());
         assert!(matches!(mt.lookup(0), LookupResult::NotInMemory));
 
-        let mut bad_proof = proof.clone();
-        if let MerkleNode::Leaf {
-            value: _,
-            pos: _,
-            elem,
-        } = &mut bad_proof.proof[0]
-        {
-            *elem = F::from(4u64);
-        } else {
-            unreachable!()
-        }
-
-        let result = mt.remember(0, elem, &bad_proof);
-        assert!(result.is_err());
-
-        let mut forge_proof = MerkleProof::new(2, proof.proof.clone());
-        if let MerkleNode::Leaf {
-            value: _,
-            pos,
-            elem,
-        } = &mut forge_proof.proof[0]
-        {
-            *pos = 2;
-            *elem = F::from(0u64);
-        } else {
-            unreachable!()
-        }
-        let result = mt.remember(2, elem, &forge_proof);
-        assert!(result.is_err());
+        // Wrong element
+        assert!(mt.remember(0, F::from(19u64), &proof).is_err());
+        // Wrong pos
+        assert!(mt.remember(1, elem, &proof).is_err());
+        // Wrong proof
+        lookup_proof.0[0][0] = F::one();
+        assert!(mt.remember(0, elem, &lookup_proof).is_err());
 
         assert!(mt.remember(0, elem, &proof).is_ok());
         assert!(mt.lookup(0).expect_ok().is_ok());
@@ -277,8 +279,7 @@ mod mt_tests {
     fn test_mt_serde_helper<F: RescueParameter>() {
         let mt =
             RescueMerkleTree::<F>::from_elems(Some(2), [F::from(3u64), F::from(1u64)]).unwrap();
-        let proof = mt.lookup(0).expect_ok().unwrap().1;
-        let node = &proof.proof[0];
+        let (_, proof) = mt.lookup(0).expect_ok().unwrap();
 
         assert_eq!(
             mt,
@@ -288,10 +289,6 @@ mod mt_tests {
             proof,
             bincode::deserialize(&bincode::serialize(&proof).unwrap()).unwrap()
         );
-        assert_eq!(
-            *node,
-            bincode::deserialize(&bincode::serialize(node).unwrap()).unwrap()
-        );
     }
 
     #[test]