added prometheus metrics for blocks (#1529)

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Next release
 
+- feat: add prometheus metrics for mapping worker
 - Fix(node): Fix creating a local testnet with multiple nodes fails using only
   cli flags
 - dev: change `Vec::new` to `Vec::with_capacity` where possible.

crates/client/mapping-sync/Cargo.toml

@@ -23,8 +23,10 @@ mc-storage = { workspace = true }
 mp-digest-log = { workspace = true }
 mp-hashers = { workspace = true }
 mp-transactions = { workspace = true }
+num-traits = { workspace = true }
 pallet-starknet = { workspace = true }
 pallet-starknet-runtime-api = { workspace = true }
+prometheus-endpoint = { workspace = true }
 sc-client-api = { workspace = true }
 sp-api = { workspace = true }
 sp-blockchain = { workspace = true }

crates/client/mapping-sync/src/block_metrics.rs

@@ -0,0 +1,29 @@
+use prometheus_endpoint::prometheus::{Counter, Gauge};
+use prometheus_endpoint::{register, PrometheusError, Registry};
+
+#[derive(Clone, Debug)]
+pub struct BlockMetrics {
+    pub block_height: Gauge,
+    pub transaction_count: Counter,
+    pub event_count: Counter,
+    pub l1_gas_price_wei: Gauge,
+    pub l1_gas_price_strk: Gauge,
+}
+
+impl BlockMetrics {
+    pub fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+        Ok(Self {
+            block_height: register(Gauge::new("madara_block_height", "Gauge for madara block height")?, registry)?,
+            transaction_count: register(
+                Counter::new("madara_transaction_count", "Counter for madara transaction count")?,
+                registry,
+            )?,
+            event_count: register(Counter::new("madara_event_count", "Counter for madara event count")?, registry)?,
+            l1_gas_price_wei: register(Gauge::new("madara_l1_gas_price", "Gauge for madara l1 gas price")?, registry)?,
+            l1_gas_price_strk: register(
+                Gauge::new("madara_l1_gas_price_strk", "Gauge for madara l1 gas price in strk")?,
+                registry,
+            )?,
+        })
+    }
+}

crates/client/mapping-sync/src/lib.rs

@@ -8,6 +8,7 @@
 //! # Usage
 //! The madara node should spawn a `MappingSyncWorker` among it's services.
 
+mod block_metrics;
 mod sync_blocks;
 
 use std::marker::PhantomData;
@@ -21,12 +22,15 @@ use futures_timer::Delay;
 use log::debug;
 use mp_hashers::HasherT;
 use pallet_starknet_runtime_api::StarknetRuntimeApi;
+use prometheus_endpoint::prometheus;
 use sc_client_api::backend::{Backend, StorageProvider};
 use sc_client_api::client::ImportNotifications;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 
+use crate::block_metrics::BlockMetrics;
+
 /// The worker in charge of syncing the Madara db when it receive a new Substrate block
 pub struct MappingSyncWorker<B: BlockT, C, BE, H> {
     import_notifications: ImportNotifications<B>,
@@ -41,6 +45,8 @@ pub struct MappingSyncWorker<B: BlockT, C, BE, H> {
     have_next: bool,
     retry_times: usize,
     sync_from: <B::Header as HeaderT>::Number,
+
+    block_metrics: Option<BlockMetrics>,
 }
 
 impl<B: BlockT, C, BE, H> Unpin for MappingSyncWorker<B, C, BE, H> {}
@@ -55,7 +61,10 @@ impl<B: BlockT, C, BE, H> MappingSyncWorker<B, C, BE, H> {
         frontier_backend: Arc<mc_db::Backend<B>>,
         retry_times: usize,
         sync_from: <B::Header as HeaderT>::Number,
+        prometheus_registry: Option<prometheus::Registry>,
     ) -> Self {
+        let block_metrics =
+            prometheus_registry.and_then(|registry| block_metrics::BlockMetrics::register(&registry).ok());
         Self {
             import_notifications,
             timeout,
@@ -69,6 +78,7 @@ impl<B: BlockT, C, BE, H> MappingSyncWorker<B, C, BE, H> {
             have_next: true,
             retry_times,
             sync_from,
+            block_metrics,
         }
     }
 }
@@ -119,6 +129,7 @@ where
                 self.madara_backend.as_ref(),
                 self.retry_times,
                 self.sync_from,
+                self.block_metrics.as_ref(),
             ) {
                 Ok(have_next) => {
                     self.have_next = have_next;

crates/client/mapping-sync/src/sync_blocks.rs

@@ -2,13 +2,22 @@ use mc_rpc_core::utils::get_block_by_block_hash;
 use mp_digest_log::{find_starknet_block, FindLogError};
 use mp_hashers::HasherT;
 use mp_transactions::compute_hash::ComputeTransactionHash;
+use num_traits::FromPrimitive;
 use pallet_starknet_runtime_api::StarknetRuntimeApi;
+use prometheus_endpoint::prometheus::core::Number;
 use sc_client_api::backend::{Backend, StorageProvider};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::{Backend as _, HeaderBackend};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero};
 
-fn sync_block<B: BlockT, C, BE, H>(client: &C, backend: &mc_db::Backend<B>, header: &B::Header) -> anyhow::Result<()>
+use crate::block_metrics::BlockMetrics;
+
+fn sync_block<B: BlockT, C, BE, H>(
+    client: &C,
+    backend: &mc_db::Backend<B>,
+    header: &B::Header,
+    block_metrics: Option<&BlockMetrics>,
+) -> anyhow::Result<()>
 where
     C: HeaderBackend<B> + StorageProvider<B, BE>,
     C: ProvideRuntimeApi<B>,
@@ -49,6 +58,27 @@ where
                                 .collect(),
                         };
 
+                        if let Some(block_metrics) = block_metrics {
+                            let starknet_block = &digest_starknet_block.clone();
+                            block_metrics.block_height.set(starknet_block.header().block_number.into_f64());
+
+                            // sending f64::MIN in case we exceed f64 (highly unlikely). The min numbers will
+                            // allow dashboards to catch anomalies so that it can be investigated.
+                            block_metrics
+                                .transaction_count
+                                .inc_by(f64::from_u128(starknet_block.header().transaction_count).unwrap_or(f64::MIN));
+                            block_metrics
+                                .event_count
+                                .inc_by(f64::from_u128(starknet_block.header().event_count).unwrap_or(f64::MIN));
+                            block_metrics.l1_gas_price_wei.set(
+                                f64::from_u128(starknet_block.header().l1_gas_price.price_in_wei).unwrap_or(f64::MIN),
+                            );
+
+                            block_metrics
+                                .l1_gas_price_strk
+                                .set(starknet_block.header().l1_gas_price.price_in_strk.unwrap_or(0).into_f64());
+                        }
+
                         backend.mapping().write_hashes(mapping_commitment).map_err(|e| anyhow::anyhow!(e))
                     }
                 }
@@ -98,6 +128,7 @@ fn sync_one_block<B: BlockT, C, BE, H>(
     substrate_backend: &BE,
     madara_backend: &mc_db::Backend<B>,
     sync_from: <B::Header as HeaderT>::Number,
+    block_metrics: Option<&BlockMetrics>,
 ) -> anyhow::Result<bool>
 where
     C: ProvideRuntimeApi<B>,
@@ -139,7 +170,7 @@ where
         madara_backend.meta().write_current_syncing_tips(current_syncing_tips)?;
         Ok(true)
     } else {
-        sync_block::<_, _, _, H>(client, madara_backend, &operating_header)?;
+        sync_block::<_, _, _, H>(client, madara_backend, &operating_header, block_metrics)?;
 
         current_syncing_tips.push(*operating_header.parent_hash());
         madara_backend.meta().write_current_syncing_tips(current_syncing_tips)?;
@@ -153,6 +184,7 @@ pub fn sync_blocks<B: BlockT, C, BE, H>(
     madara_backend: &mc_db::Backend<B>,
     limit: usize,
     sync_from: <B::Header as HeaderT>::Number,
+    block_metrics: Option<&BlockMetrics>,
 ) -> anyhow::Result<bool>
 where
     C: ProvideRuntimeApi<B>,
@@ -164,7 +196,8 @@ where
     let mut synced_any = false;
 
     for _ in 0..limit {
-        synced_any = synced_any || sync_one_block::<_, _, _, H>(client, substrate_backend, madara_backend, sync_from)?;
+        synced_any = synced_any
+            || sync_one_block::<_, _, _, H>(client, substrate_backend, madara_backend, sync_from, block_metrics)?;
     }
 
     Ok(synced_any)

crates/node/src/service.rs

@@ -409,6 +409,7 @@ pub fn new_full(
             madara_backend.clone(),
             3,
             0,
+            prometheus_registry.clone(),
         )
         .for_each(|()| future::ready(())),
     );