feat: listen to storage events and build the commitment state diff

Cargo.toml

@@ -23,6 +23,7 @@ members = [
   "crates/client/snos-feeder",
   "crates/client/storage",
   "crates/client/transaction-pool",
+  "crates/client/snos-feeder",
   "starknet-rpc-test",
 ]
 # All previous except for `starknet-rpc-test`
@@ -50,6 +51,7 @@ default-members = [
   "crates/client/snos-feeder",
   "crates/client/storage",
   "crates/client/transaction-pool",
+  "crates/client/snos-feeder",
 ]
 
 [profile.release]
@@ -169,6 +171,7 @@ mc-rpc-core = { path = "crates/client/rpc-core" }
 mc-block-proposer = { path = "crates/client/block-proposer" }
 mc-transaction-pool = { path = "crates/client/transaction-pool" }
 mc-data-availability = { path = "crates/client/data-availability" }
+mc-snos = { path = "crates/client/snos-feeder" }
 
 # Madara runtime
 madara-runtime = { path = "crates/runtime" }

crates/client/data-availability/Cargo.toml

@@ -21,7 +21,6 @@ jsonrpsee = { version = "0.20.0", features = [
   "ws-client",
   "macros",
 ] }
-lazy_static = { workspace = true }
 log = "0.4.19"
 reqwest = { version = "0.11.18", features = ["blocking", "json"] }
 serde = { workspace = true }
@@ -36,7 +35,6 @@ sc-client-api = { workspace = true }
 sp-api = { workspace = true }
 sp-blockchain = { workspace = true }
 sp-core = { workspace = true }
-sp-io = { workspace = true }
 sp-runtime = { workspace = true }
 
 # Starknet

crates/client/data-availability/src/lib.rs

@@ -12,6 +12,7 @@ use anyhow::Result;
 use async_trait::async_trait;
 use ethers::types::{I256, U256};
 use futures::StreamExt;
+use mp_storage::{SN_NONCE_PREFIX, SN_STORAGE_PREFIX};
 use sc_client_api::client::BlockchainEvents;
 use serde::Deserialize;
 use sp_api::ProvideRuntimeApi;
@@ -94,13 +95,13 @@ where
                     key = raw_split.1;
                 }
 
-                if prefix == *utils::SN_NONCE_PREFIX {
+                if prefix == *SN_NONCE_PREFIX {
                     if let Some(data) = event.2 {
                         nonces.insert(key, data.0.as_slice());
                     }
                 }
 
-                if prefix == *utils::SN_STORAGE_PREFIX {
+                if prefix == *SN_STORAGE_PREFIX {
                     if let Some(data) = event.2 {
                         // first 32 bytes = contract address, second 32 bytes = storage variable
                         let write_split = key.split_at(32);

crates/client/data-availability/src/utils.rs

@@ -1,22 +1,8 @@
 use std::collections::HashMap;
 
 use ethers::types::U256;
-use lazy_static::lazy_static;
-use mp_storage::{
-    PALLET_STARKNET, STARKNET_CONTRACT_CLASS, STARKNET_CONTRACT_CLASS_HASH, STARKNET_NONCE, STARKNET_STORAGE,
-};
-use sp_io::hashing::twox_128;
 use url::{ParseError, Url};
 
-lazy_static! {
-    pub static ref SN_NONCE_PREFIX: Vec<u8> = [twox_128(PALLET_STARKNET), twox_128(STARKNET_NONCE)].concat();
-    pub static ref SN_CONTRACT_CLASS_HASH_PREFIX: Vec<u8> =
-        [twox_128(PALLET_STARKNET), twox_128(STARKNET_CONTRACT_CLASS_HASH)].concat();
-    pub static ref SN_CONTRACT_CLASS_PREFIX: Vec<u8> =
-        [twox_128(PALLET_STARKNET), twox_128(STARKNET_CONTRACT_CLASS)].concat();
-    pub static ref SN_STORAGE_PREFIX: Vec<u8> = [twox_128(PALLET_STARKNET), twox_128(STARKNET_STORAGE)].concat();
-}
-
 // encode calldata:
 // - https://docs.starknet.io/documentation/architecture_and_concepts/Data_Availability/on-chain-data/#pre_v0.11.0_example
 pub fn pre_0_11_0_state_diff(

crates/client/snos-feeder/Cargo.toml

@@ -1,12 +1,28 @@
 [package]
-name = "snos-feeder"
+name = "mc-snos"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
 version.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
+# Substrate
+starknet_api = { workspace = true, features = ["std"] }
+sp-runtime = { workspace = true, features = ["std"] }
+sp-blockchain = { workspace = true }
+sp-api = { workspace = true, features = ["std"] }
+sp-storage = { workspace = true, features = ["std"] }
+sc-client-api = { workspace = true }
+
+# Madara
+mp-storage = { workspace = true, features = ["std"] }
+mp-digest-log = { workspace = true, features = ["std"] }
+pallet-starknet = { workspace = true }
+
+# Starknet
 snos = {git = "https://github.com/keep-starknet-strange/snos", branch = "main" }
 blockifier = { workspace = true, features = ["std"] }
+
+log = { workspace = true }
+indexmap = { workspace = true }
+futures = "0.3.21"

crates/client/snos-feeder/src/lib.rs

@@ -1 +1,110 @@
+use std::marker::PhantomData;
+use std::sync::Arc;
 
+use futures::StreamExt;
+use indexmap::IndexMap;
+use mp_storage::{SN_COMPILED_CLASS_HASH_PREFIX, SN_CONTRACT_CLASS_HASH_PREFIX, SN_NONCE_PREFIX, SN_STORAGE_PREFIX};
+use pallet_starknet::runtime_api::StarknetRuntimeApi;
+use sc_client_api::client::BlockchainEvents;
+use sp_api::ProvideRuntimeApi;
+use sp_blockchain::HeaderBackend;
+use sp_runtime::traits::Block as BlockT;
+use starknet_api::api_core::{ClassHash, CompiledClassHash, ContractAddress, Nonce, PatriciaKey};
+use starknet_api::hash::StarkFelt;
+use starknet_api::state::StorageKey as StarknetStorageKey;
+
+pub struct SnosWorker<B, C>(PhantomData<(B, C)>);
+
+impl<B, C> SnosWorker<B, C>
+where
+    B: BlockT,
+    C: ProvideRuntimeApi<B>,
+    C::Api: StarknetRuntimeApi<B>,
+    C: BlockchainEvents<B> + 'static,
+    C: HeaderBackend<B>,
+{
+    pub async fn run_snos(client: Arc<C>) {
+        let mut storage_event_st = client
+            // https://github.com/paritytech/polkadot-sdk/issues/1989
+            // I can't find a way to make the child_keys logic work
+            .storage_changes_notification_stream(None, None)
+            .expect("the node storage changes notification stream should be up and running");
+
+        while let Some(block_change_set) = storage_event_st.next().await {
+            let block_context = match client.runtime_api().get_block_context(block_change_set.block) {
+                Ok(bc) => bc,
+                Err(e) => {
+                    log::error!(
+                        "failed to retrieve the block context from the starknet pallet runtime. Abort SNOS execution \
+                         for this block. Error: {e}",
+                    );
+                    continue;
+                }
+            };
+
+            let mut commitment_state_diff = blockifier::state::cached_state::CommitmentStateDiff {
+                address_to_class_hash: Default::default(),
+                address_to_nonce: Default::default(),
+                storage_updates: Default::default(),
+                class_hash_to_compiled_class_hash: Default::default(),
+            };
+
+            for (_prefix, full_storage_key, change) in block_change_set.changes.iter() {
+                // The storages we are interested in here all have longe keys
+                if full_storage_key.0.len() < 32 {
+                    continue;
+                }
+                let prefix = &full_storage_key.0[..32];
+
+                // All the try_into are safe to unwrap because we know that is what the storage contains
+                // and therefore what length it is
+                if prefix == *SN_NONCE_PREFIX {
+                    let contract_address =
+                        ContractAddress(PatriciaKey(StarkFelt(full_storage_key.0[32..].try_into().unwrap())));
+                    // `change` is safe to unwrap as `Nonces` storage is `ValueQuery`
+                    let nonce = Nonce(StarkFelt(change.unwrap().0.clone().try_into().unwrap()));
+                    commitment_state_diff.address_to_nonce.insert(contract_address, nonce);
+                } else if prefix == *SN_STORAGE_PREFIX {
+                    let contract_address =
+                        ContractAddress(PatriciaKey(StarkFelt(full_storage_key.0[32..64].try_into().unwrap())));
+                    let storage_key =
+                        StarknetStorageKey(PatriciaKey(StarkFelt(full_storage_key.0[64..].try_into().unwrap())));
+                    // `change` is safe to unwrap as `StorageView` storage is `ValueQuery`
+                    let value = StarkFelt(change.unwrap().0.clone().try_into().unwrap());
+
+                    match commitment_state_diff.storage_updates.get_mut(&contract_address) {
+                        Some(contract_storage) => {
+                            contract_storage.insert(storage_key, value);
+                        }
+                        None => {
+                            let mut contract_storage: IndexMap<_, _, _> = Default::default();
+                            contract_storage.insert(storage_key, value);
+
+                            commitment_state_diff.storage_updates.insert(contract_address, contract_storage);
+                        }
+                    }
+                } else if prefix == *SN_CONTRACT_CLASS_HASH_PREFIX {
+                    let contract_address =
+                        ContractAddress(PatriciaKey(StarkFelt(full_storage_key.0[32..].try_into().unwrap())));
+                    // `change` is safe to unwrap as `ContractClassHashes` storage is `ValueQuery`
+                    let class_hash = ClassHash(StarkFelt(change.unwrap().0.clone().try_into().unwrap()));
+
+                    commitment_state_diff.address_to_class_hash.insert(contract_address, class_hash);
+                } else if prefix == *SN_COMPILED_CLASS_HASH_PREFIX {
+                    let class_hash = ClassHash(StarkFelt(full_storage_key.0[32..].try_into().unwrap()));
+                    // `change` is safe to unwrap, despite `CompiledClassHashes` being an `OptionQuery`,
+                    // because the starknet protocol guarantee that its storage values
+                    // are never erased (set to `None` again)
+                    let compiled_class_hash =
+                        CompiledClassHash(StarkFelt(change.unwrap().0.clone().try_into().unwrap()));
+
+                    commitment_state_diff.class_hash_to_compiled_class_hash.insert(class_hash, compiled_class_hash);
+                }
+            }
+
+            println!("commitment: {:?}, block context: {:?}", commitment_state_diff, block_context);
+
+            // TODO: call snos
+        }
+    }
+}

crates/node/Cargo.toml

@@ -84,6 +84,7 @@ mc-storage = { workspace = true }
 mc-transaction-pool = { workspace = true }
 pallet-starknet = { workspace = true }
 starknet-core = { workspace = true }
+mc-snos = { workspace = true }
 
 # Primitives
 mp-block = { workspace = true }

crates/node/src/service.rs

@@ -20,6 +20,7 @@ use mc_data_availability::ethereum::config::EthereumConfig;
 use mc_data_availability::ethereum::EthereumClient;
 use mc_data_availability::{DaClient, DaLayer, DataAvailabilityWorker};
 use mc_mapping_sync::MappingSyncWorker;
+use mc_snos::SnosWorker;
 use mc_storage::overrides_handle;
 use mc_transaction_pool::FullPool;
 use mp_sequencer_address::{
@@ -386,6 +387,8 @@ pub fn new_full(
         .for_each(|()| future::ready(())),
     );
 
+    task_manager.spawn_essential_handle().spawn("snos", Some("madara"), SnosWorker::run_snos(client.clone()));
+
     // initialize data availability worker
     if let Some((da_layer, da_path)) = da_layer {
         let da_client: Box<dyn DaClient + Send + Sync> = match da_layer {

crates/primitives/storage/Cargo.toml

@@ -16,9 +16,11 @@ parity-scale-codec = { workspace = true, features = [
   "derive",
 ], optional = true }
 serde = { workspace = true, optional = true, features = ["derive"] }
+lazy_static = { workspace = true }
+sp-io = { workspace = true }
 
 [features]
 default = ["std"]
-std = ["serde?/std", "parity-scale-codec?/std"]
+std = ["serde?/std", "parity-scale-codec?/std", "sp-io/std"]
 serde = ["dep:serde"]
 parity-scale-codec = ["dep:parity-scale-codec"]

crates/primitives/storage/src/lib.rs

@@ -1,6 +1,12 @@
 //! StarkNet storage primitives.
 #![cfg_attr(not(feature = "std"), no_std)]
 
+extern crate alloc;
+use alloc::vec::Vec;
+
+use lazy_static::lazy_static;
+use sp_io::hashing::twox_128;
+
 /// Current version of pallet Starknet's storage schema is stored under this key.
 pub const PALLET_STARKNET_SCHEMA: &[u8] = b":starknet_schema";
 
@@ -23,6 +29,19 @@ pub const STARKNET_CONTRACT_CLASS: &[u8] = b"ContractClasses";
 pub const STARKNET_NONCE: &[u8] = b"Nonces";
 /// Starknet storage
 pub const STARKNET_STORAGE: &[u8] = b"StorageView";
+/// Compiled class hashes
+pub const STARKNET_COMPILED_CLASS_HASH: &[u8] = b"CompiledClassHashes";
+
+lazy_static! {
+    pub static ref SN_NONCE_PREFIX: Vec<u8> = [twox_128(PALLET_STARKNET), twox_128(STARKNET_NONCE)].concat();
+    pub static ref SN_CONTRACT_CLASS_HASH_PREFIX: Vec<u8> =
+        [twox_128(PALLET_STARKNET), twox_128(STARKNET_CONTRACT_CLASS_HASH)].concat();
+    pub static ref SN_CONTRACT_CLASS_PREFIX: Vec<u8> =
+        [twox_128(PALLET_STARKNET), twox_128(STARKNET_CONTRACT_CLASS)].concat();
+    pub static ref SN_STORAGE_PREFIX: Vec<u8> = [twox_128(PALLET_STARKNET), twox_128(STARKNET_STORAGE)].concat();
+    pub static ref SN_COMPILED_CLASS_HASH_PREFIX: Vec<u8> =
+        [twox_128(PALLET_STARKNET), twox_128(STARKNET_COMPILED_CLASS_HASH)].concat();
+}
 
 /// The schema version for Pallet Starknet's storage.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]