diff --git a/backend/Cargo.lock b/backend/Cargo.lock index cb87b3c89..869b3725c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -58,9 +58,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -171,6 +171,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + [[package]] name = "base32" version = "0.4.0" @@ -595,6 +604,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -610,6 +625,21 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -654,6 +684,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -672,8 +713,10 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1033,6 +1076,8 @@ dependencies = [ "hex", "hmac", "jsonwebtoken", + "once_cell", + "prometheus", "rand 0.8.6", "redis", "rust_decimal", @@ -1136,6 +1181,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1456,6 +1507,52 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags", + "hex", + "lazy_static", + "procfs-core", + "rustix 0.38.44", +] + +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags", + "hex", +] + +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "protobuf", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "ptr_meta" version = "0.1.4" @@ -1570,8 +1667,10 @@ checksum = "09d8f99a4090c89cc489a94833c901ead69bfbf3877b4867d5482e321ee875bc" dependencies = [ "arc-swap", "async-trait", + "backon", "bytes", "combine", + "futures", "futures-util", "itertools", "itoa", @@ -1719,6 +1818,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1728,7 +1840,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -2306,7 +2418,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 666849c6e..835f6ab6f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] tokio = { version = "1.0", features = ["full"] } +redis = { version = "0.27", features = ["tokio-comp", "connection-manager"] } axum = { version = "0.8", features = ["json", "multipart", "macros", "ws"] } tower = { version = "0.5", features = ["util"] } tower-http = { version = "0.6", features = ["cors", "trace", "request-id", "util"] } @@ -28,4 +29,5 @@ jsonwebtoken = "9.0" base64 = "0.21" stellar-strkey = "0.0.8" ed25519-dalek = { version = "2.1", features = ["pkcs8", "rand_core"] } -redis = { version = "0.27", features = ["tokio-comp"] } +prometheus = { version = "0.13", features = ["process"] } +once_cell = "1.19" diff --git a/backend/src/api.rs b/backend/src/api.rs index cfd9a2cae..33a7f6288 100644 --- a/backend/src/api.rs +++ b/backend/src/api.rs @@ -18,6 +18,7 @@ use uuid::Uuid; use crate::auth::signature_auth_middleware; use crate::cache::PlanCache; use crate::kyc_webhook::kyc_webhook_handler; +use crate::metrics::{latency_middleware, metrics_handler}; use crate::stellar_anchor::AnchorRegistry; use crate::ws::{ws_handler, KycUpdateEvent}; use crate::yield_calculator; @@ -136,6 +137,8 @@ pub fn create_router(state: Arc) -> Router { Router::new() .merge(user_routes) .merge(public_routes) + .route("/metrics", get(metrics_handler)) + .layer(from_fn(latency_middleware)) .layer(cors) .with_state(state) } diff --git a/backend/src/lib.rs b/backend/src/lib.rs index c75cf7567..cf3cc970c 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -5,6 +5,7 @@ pub mod config; pub mod db; pub mod inactivity_watchdog; pub mod kyc_webhook; +pub mod metrics; pub mod stellar_anchor; pub mod telemetry; pub mod ws; diff --git a/backend/src/main.rs b/backend/src/main.rs index 074415c14..1ffb6ec7e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1,5 +1,5 @@ use inheritx_backend::{ - create_router, telemetry, AppState, Config, DbManager, InactivityWatchdogConfig, + create_router, metrics, telemetry, AppState, Config, DbManager, InactivityWatchdogConfig, InactivityWatchdogService, }; use std::net::SocketAddr; @@ -11,6 +11,9 @@ async fn main() -> Result<(), Box> { // Initialize tracing logging telemetry::init_tracing()?; + // Initialize Prometheus metrics + metrics::init(); + //loading the .env dotenvy::dotenv().ok(); @@ -65,6 +68,18 @@ async fn main() -> Result<(), Box> { )); inactivity_watchdog.start(); + // Periodically refresh DB pool metrics + { + let pool = db_pool.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); + loop { + interval.tick().await; + metrics::update_db_pool_metrics(&pool); + } + }); + } + // Create Axum application let app = create_router(state); diff --git a/backend/src/metrics.rs b/backend/src/metrics.rs new file mode 100644 index 000000000..4db1555c2 --- /dev/null +++ b/backend/src/metrics.rs @@ -0,0 +1,104 @@ +use axum::{extract::Request, http::StatusCode, middleware::Next, response::IntoResponse}; +use once_cell::sync::Lazy; +use prometheus::{ + histogram_opts, opts, register_gauge, register_histogram_vec, Encoder, Gauge, HistogramVec, + TextEncoder, +}; +use std::time::Instant; + +/// Tracks number of in-flight HTTP connections. +pub static ACTIVE_CONNECTIONS: Lazy = Lazy::new(|| { + register_gauge!(opts!( + "inheritx_active_connections", + "Number of currently active HTTP connections" + )) + .expect("failed to register active_connections gauge") +}); + +/// Per-route request latency histogram (seconds). +/// Labels: method, path, status +pub static REQUEST_LATENCY: Lazy = Lazy::new(|| { + register_histogram_vec!( + histogram_opts!( + "inheritx_http_request_duration_seconds", + "HTTP request latency in seconds", + // Buckets tuned for low-latency API; p95/p99 computed by Prometheus from these. + vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] + ), + &["method", "path", "status"] + ) + .expect("failed to register request_latency histogram") +}); + +/// DB pool size (total connections in the pool). +pub static DB_POOL_SIZE: Lazy = Lazy::new(|| { + register_gauge!(opts!( + "inheritx_db_pool_size", + "Total connections in the DB pool" + )) + .expect("failed to register db_pool_size gauge") +}); + +/// DB pool idle connections. +pub static DB_POOL_IDLE: Lazy = Lazy::new(|| { + register_gauge!(opts!( + "inheritx_db_pool_idle", + "Idle connections in the DB pool" + )) + .expect("failed to register db_pool_idle gauge") +}); + +/// Call once at startup to force lazy initialization of all metrics. +pub fn init() { + Lazy::force(&ACTIVE_CONNECTIONS); + Lazy::force(&REQUEST_LATENCY); + Lazy::force(&DB_POOL_SIZE); + Lazy::force(&DB_POOL_IDLE); +} + +/// Updates DB pool gauges from the current sqlx pool state. +pub fn update_db_pool_metrics(pool: &sqlx::PgPool) { + DB_POOL_SIZE.set(pool.size() as f64); + DB_POOL_IDLE.set(pool.num_idle() as f64); +} + +/// GET /metrics — Prometheus text exposition. +pub async fn metrics_handler() -> impl IntoResponse { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buf = Vec::new(); + if encoder.encode(&metric_families, &mut buf).is_err() { + return (StatusCode::INTERNAL_SERVER_ERROR, "encoding error").into_response(); + } + ( + StatusCode::OK, + [(axum::http::header::CONTENT_TYPE, encoder.format_type())], + buf, + ) + .into_response() +} + +/// Axum middleware: tracks active connections and records per-route latency. +pub async fn latency_middleware(req: Request, next: Next) -> impl IntoResponse { + ACTIVE_CONNECTIONS.inc(); + + let method = req.method().to_string(); + // Use the matched path pattern (e.g. "/api/plans") to avoid high cardinality. + let path = req + .extensions() + .get::() + .map(|p| p.as_str().to_owned()) + .unwrap_or_else(|| req.uri().path().to_owned()); + + let start = Instant::now(); + let response = next.run(req).await; + let elapsed = start.elapsed().as_secs_f64(); + + let status = response.status().as_u16().to_string(); + REQUEST_LATENCY + .with_label_values(&[&method, &path, &status]) + .observe(elapsed); + + ACTIVE_CONNECTIONS.dec(); + response +}