From 4b58bf235d6242286061801a86b73bdfce3b017c Mon Sep 17 00:00:00 2001 From: Ibinola Date: Thu, 25 Jun 2026 14:18:20 +0100 Subject: [PATCH 1/2] Implement devkit analysis issues #258-#261 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add percentile_nearest and percentile_interpolated free functions (#259, #260) - Implement PsquaredEstimator with P² streaming percentile algorithm (#261) - Document ## Simulation section in README with FeeModelConfig, NetworkLoadConfig, example usage, and CSV export instructions (#258) Closes #258 Closes #259 Closes #260 Closes #261 --- packages/devkit/README.md | 102 ++++++-- packages/devkit/src/analysis/percentile.rs | 276 +++++++++++++++++++++ 2 files changed, 364 insertions(+), 14 deletions(-) diff --git a/packages/devkit/README.md b/packages/devkit/README.md index 5011af9..aa96212 100644 --- a/packages/devkit/README.md +++ b/packages/devkit/README.md @@ -27,33 +27,107 @@ Developer toolkit for the Stellar Fee Tracker. Provides utilities for testing, m ## Simulation -The `simulation` module provides fee modelling and network-load generation without any live-network dependencies. +The `simulation` module provides fee modelling, network-load generation, and congestion prediction without any live-network dependencies. ### `FeeModelConfig` fields -| Field | Type | Description | -|---|---|---| -| `base_fee` | `u64` | Minimum fee (stroops) used as the simulation floor | -| `surge_multiplier` | `f64` | Fee multiplier applied when the network is congested | -| `congestion_threshold` | `f64` | Load ratio (0.0–1.0) above which surge pricing activates | +| Field | Type | Default | Description | +|---|---|---|---| +| `base_fee` | `u64` | `100` | Base fee in stroops | +| `spike_probability` | `f64` | `0.05` | Probability that any given ledger is a spike (0.0–1.0) | +| `spike_multiplier` | `u64` | `10` | Multiplier applied to `base_fee` during a spike | +| `ledger_interval_secs` | `u64` | `5` | Seconds between simulated ledgers | +| `ledger_count` | `u64` | `100` | Number of ledgers to generate per `run()` call | +| `seed` | `Option` | `None` | RNG seed for reproducible output | +| `noise_factor` | `f64` | `0.0` | Gaussian noise stddev as a fraction of `base_fee` | + +### `NetworkLoadConfig` fields + +| Field | Type | Default | Description | +|---|---|---|---| +| `min_tx` | `u64` | `10` | Minimum transactions per ledger | +| `max_tx` | `u64` | `1000` | Maximum transactions per ledger | +| `ledger_capacity` | `u64` | `1000` | Maximum tx capacity per ledger | +| `ledger_interval_ms` | `u64` | `5000` | Time between ledger closes in ms | +| `seed` | `Option` | `None` | RNG seed for reproducibility | ### Example usage ```rust -use stellar_devkit::simulation::{FeeModel, NetworkLoad}; +use stellar_devkit::simulation::fee_model::{FeeModel, FeeModelConfig}; +use stellar_devkit::simulation::network_load::{NetworkLoad, NetworkLoadConfig}; +use stellar_devkit::simulation::congestion_predictor::{CongestionPredictor, CongestionInput, congestion_label}; + +// Configure a fee scenario +let fee_cfg = FeeModelConfig { + base_fee: 100, + spike_probability: 0.1, + spike_multiplier: 5, + seed: Some(42), + ..FeeModelConfig::default() +}; -let load = NetworkLoad::constant(0.85); // 85 % utilisation -let result = FeeModel::run(&load, base_fee: 100, surge_multiplier: 2.0, congestion_threshold: 0.8); -println!("recommended fee: {} stroops", result.recommended_fee); +// Generate fee points +let points = FeeModel::run(&fee_cfg); +println!("Generated {} fee points", points.len()); + +// Configure network load +let load_cfg = NetworkLoadConfig { + min_tx: 50, + max_tx: 800, + ledger_capacity: 1000, + seed: Some(7), + ..NetworkLoadConfig::default() +}; +let mut load = NetworkLoad::new(load_cfg); +let ledgers = load.simulate(10); + +// Predict congestion +let label = congestion_label(&CongestionInput { + recent_fee_window: 250.0, + capacity_usage: 0.75, + spike_count: 3, +}); +println!("Congestion: {:?}", label); ``` -### Output format (`SimResult`) +### Output format (`FeePoint`) + +Each `FeePoint` represents a single simulated ledger: | Field | Type | Description | |---|---|---| -| `recommended_fee` | `u64` | Suggested fee for the simulated conditions | -| `congested` | `bool` | Whether surge pricing was triggered | -| `load_ratio` | `f64` | Network utilisation at simulation time | +| `timestamp` | `u64` | Simulated Unix timestamp (seconds) | +| `fee` | `u64` | Fee in stroops for this ledger | +| `ledger` | `u64` | Ledger sequence number (1-based) | +| `is_spike` | `bool` | Whether this ledger was a spike | + +### CSV export + +Fee points can be exported to CSV via the CLI: + +```bash +cargo run --bin devkit -- export ./fees.db --output fees.csv +``` + +The CSV format matches the `FeePoint` shape: + +``` +timestamp,fee,ledger,is_spike +1700000000,100,1,false +1700000005,500,2,true +1700000010,110,3,false +``` + +For programmatic export, serialise `FeePoint` slices directly: + +```rust +use stellar_devkit::simulation::fee_model::{FeeModel, FeeModelConfig, FeeCurve}; + +let points = FeeModel::run(&FeeModelConfig::default()); +let json = FeeCurve::fee_points_to_json(&points, 100)?; +println!("{}", json); +``` ## Running diff --git a/packages/devkit/src/analysis/percentile.rs b/packages/devkit/src/analysis/percentile.rs index da13085..b11fb14 100644 --- a/packages/devkit/src/analysis/percentile.rs +++ b/packages/devkit/src/analysis/percentile.rs @@ -70,6 +70,191 @@ impl Percentile { } } +// ── Free functions (Issue #259, #260) ──────────────────────────────────────── + +/// Nearest-rank percentile of a pre-sorted slice. +/// Returns 0 for empty slices. `p` must be in 0..=100. +pub fn percentile_nearest(sorted: &[u64], p: u8) -> u64 { + Percentile::nearest_rank(sorted, p as usize) +} + +/// Linear-interpolation percentile of a pre-sorted slice. +/// Returns 0.0 for empty slices. `p` must be in 0..=100. +pub fn percentile_interpolated(sorted: &[u64], p: u8) -> f64 { + if sorted.is_empty() { + return 0.0; + } + if sorted.len() == 1 { + return sorted[0] as f64; + } + let rank = (p as f64 / 100.0) * (sorted.len() - 1) as f64; + let lo = rank.floor() as usize; + let hi = rank.ceil() as usize; + let frac = rank - lo as f64; + sorted[lo] as f64 + frac * (sorted[hi] as f64 - sorted[lo] as f64) +} + +// ── P² streaming percentile estimator (Issue #261) ─────────────────────────── + +/// Streaming percentile estimator using the P² algorithm (Jain & Chlamtac, 1985). +/// +/// Estimates a single percentile without storing all observations. Uses five +/// markers that track the distribution shape as values arrive. +/// +/// # Example +/// +/// ``` +/// use stellar_devkit::analysis::percentile::PsquaredEstimator; +/// +/// let mut est = PsquaredEstimator::new(0.50); +/// for v in 1..=1000 { +/// est.update(v); +/// } +/// let p50 = est.estimate(); +/// assert!((400.0..=600.0).contains(&p50), "p50={p50} not near 500"); +/// ``` +#[derive(Debug, Clone)] +pub struct PsquaredEstimator { + p: f64, + count: usize, + markers: [(f64, f64); 5], + init_vals: [u64; 5], + init_count: usize, +} + +impl PsquaredEstimator { + /// Create a new estimator for the given percentile. + /// + /// `p` must be in `(0.0, 1.0)`. Common values: `0.50` (p50), `0.90` (p90), + /// `0.95` (p95), `0.99` (p99). + /// + /// # Panics + /// + /// Panics if `p` is not in `(0.0, 1.0)`. + pub fn new(p: f64) -> Self { + assert!(p > 0.0 && p < 1.0, "p must be in (0.0, 1.0), got {p}"); + Self { + p, + count: 0, + markers: [(0.0, 0.0); 5], + init_vals: [0; 5], + init_count: 0, + } + } + + /// Feed a new observation into the estimator. + pub fn update(&mut self, x: u64) { + if self.count < 5 { + self.init_vals[self.init_count] = x; + self.init_count += 1; + self.count += 1; + if self.count == 5 { + self.init_markers(); + } + return; + } + + self.count += 1; + let x = x as f64; + let n = self.count as f64; + + // Find cell k where the new value belongs + let mut k = 0; + for i in 0..4 { + if x >= self.markers[i].1 { + k = i + 1; + } + } + + // Update extreme markers if outside range + if x < self.markers[0].1 { + self.markers[0].1 = x; + } + if x > self.markers[4].1 { + self.markers[4].1 = x; + } + + // Increment marker positions from k+1 to 4 (paper: j = k+1..4) + // When k=0 (x < h₀), we skip marker 0 so its n stays at 1. + // When k=4 (x ≥ h₃), we only increment marker 4. + let start = if k == 0 { 1 } else { k }; + for i in start..5 { + self.markers[i].0 += 1.0; + } + + // Update desired positions using the P² quantile mapping + let n_prime = |i: usize| match i { + 0 => 1.0, + 1 => 1.0 + self.p * (n - 1.0) / 2.0, + 2 => 1.0 + self.p * (n - 1.0), + 3 => 1.0 + (1.0 + self.p) * (n - 1.0) / 2.0, + 4 => n, + _ => unreachable!(), + }; + + // Adjust interior markers (i = 1,2,3) + for i in 1..4 { + let d = n_prime(i) - self.markers[i].0; + let (n_im1, h_im1) = self.markers[i - 1]; + let (n_i, h_i) = self.markers[i]; + let (n_ip1, h_ip1) = self.markers[i + 1]; + + let cond = (d >= 1.0 && (n_ip1 - n_i).abs() > 1.0) + || (d <= -1.0 && (n_im1 - n_i).abs() > 1.0); + if !cond { + continue; + } + + // Parabolic update for h_i (canonical P² formula) + let d_i = (n_i - n_im1 + d) * (h_ip1 - h_i) / (n_ip1 - n_i) + + (n_ip1 - n_i - d) * (h_i - h_im1) / (n_i - n_im1); + let h_prime = h_i + d / (n_ip1 - n_im1) * d_i; + + let new_h = if h_im1 < h_prime && h_prime < h_ip1 { + h_prime + } else { + // Linear fallback: step proportionally within the containing cell + if d >= 0.0 { + h_i + (h_ip1 - h_i) / (n_ip1 - n_i) + } else { + h_i + (h_im1 - h_i) / (n_im1 - n_i) + } + }; + + let new_n = n_i + if d >= 0.0 { 1.0 } else { -1.0 }; + self.markers[i] = (new_n, new_h); + } + } + + /// Return the current estimate for the configured percentile. + pub fn estimate(&self) -> f64 { + match self.count { + 0 => 0.0, + 1..=4 => { + self.init_vals[..self.init_count].iter().sum::() as f64 / self.init_count as f64 + } + _ => self.markers[2].1, + } + } + + /// Return the number of observations seen so far. + pub fn count(&self) -> usize { + self.count + } + + fn init_markers(&mut self) { + self.init_vals.sort_unstable(); + let p = self.p; + self.markers = [ + (1.0, self.init_vals[0] as f64), + (1.0 + 2.0 * p, self.init_vals[1] as f64), + (1.0 + 4.0 * p, self.init_vals[2] as f64), + (1.0 + 6.0 * p, self.init_vals[3] as f64), + (5.0, self.init_vals[4] as f64), + ]; + } +} + #[cfg(test)] mod tests { use super::*; @@ -115,4 +300,95 @@ mod tests { fn fee_distribution_summary_empty() { assert!(Percentile::fee_distribution_summary(&[]).is_none()); } + + // ── Free function tests (Issue #259, #260) ─────────────────────────────── + + #[test] + fn percentile_nearest_free_function() { + let data = [10, 20, 30, 40, 50]; + assert_eq!(percentile_nearest(&data, 50), 30); + assert_eq!(percentile_nearest(&data, 100), 50); + assert_eq!(percentile_nearest(&data, 1), 10); + } + + #[test] + fn percentile_nearest_empty() { + assert_eq!(percentile_nearest(&[], 50), 0); + } + + #[test] + fn percentile_interpolated_free_function() { + let data = [10, 20, 30, 40, 50]; + assert!((percentile_interpolated(&data, 50) - 30.0).abs() < 1e-9); + assert!((percentile_interpolated(&data, 0) - 10.0).abs() < 1e-9); + assert!((percentile_interpolated(&data, 100) - 50.0).abs() < 1e-9); + } + + #[test] + fn percentile_interpolated_empty() { + assert!((percentile_interpolated(&[], 50) - 0.0).abs() < 1e-9); + } + + #[test] + fn percentile_interpolated_two_elements_midpoint() { + let data = [10, 20]; + assert!((percentile_interpolated(&data, 50) - 15.0).abs() < 1e-9); + } + + // ── P² estimator tests (Issue #261) ────────────────────────────────────── + + #[test] + fn psquared_estimate_before_5_observations_returns_mean() { + let mut est = PsquaredEstimator::new(0.95); + est.update(10); + est.update(20); + assert!((est.estimate() - 15.0).abs() < 1e-9); + assert_eq!(est.count(), 2); + } + + #[test] + fn psquared_empty_estimate_is_zero() { + let est = PsquaredEstimator::new(0.95); + assert!((est.estimate() - 0.0).abs() < 1e-9); + assert_eq!(est.count(), 0); + } + + #[test] + fn psquared_converges_for_uniform_data() { + let mut est = PsquaredEstimator::new(0.50); + for v in 1..=1000 { + est.update(v); + } + let p50 = est.estimate(); + assert!( + (400.0..=600.0).contains(&p50), + "p50={p50} not near 500" + ); + } + + #[test] + fn psquared_p95_on_simple_sequence() { + let mut est = PsquaredEstimator::new(0.95); + for v in 1..=200 { + est.update(v); + } + let p95 = est.estimate(); + assert!( + (175.0..=200.0).contains(&p95), + "p95={p95} not in [175, 200]" + ); + } + + #[test] + fn psquared_p99_on_simple_sequence() { + let mut est = PsquaredEstimator::new(0.99); + for v in 1..=500 { + est.update(v); + } + let p99 = est.estimate(); + assert!( + (480.0..=505.0).contains(&p99), + "p99={p99} not near 500" + ); + } } From 285e3970fef8f9850b02e345e8f75a66168b5f0e Mon Sep 17 00:00:00 2001 From: Ibinola Date: Thu, 25 Jun 2026 14:24:01 +0100 Subject: [PATCH 2/2] cargo fmt --- packages/devkit/src/analysis/percentile.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/devkit/src/analysis/percentile.rs b/packages/devkit/src/analysis/percentile.rs index b11fb14..271f003 100644 --- a/packages/devkit/src/analysis/percentile.rs +++ b/packages/devkit/src/analysis/percentile.rs @@ -199,8 +199,8 @@ impl PsquaredEstimator { let (n_i, h_i) = self.markers[i]; let (n_ip1, h_ip1) = self.markers[i + 1]; - let cond = (d >= 1.0 && (n_ip1 - n_i).abs() > 1.0) - || (d <= -1.0 && (n_im1 - n_i).abs() > 1.0); + let cond = + (d >= 1.0 && (n_ip1 - n_i).abs() > 1.0) || (d <= -1.0 && (n_im1 - n_i).abs() > 1.0); if !cond { continue; } @@ -231,7 +231,8 @@ impl PsquaredEstimator { match self.count { 0 => 0.0, 1..=4 => { - self.init_vals[..self.init_count].iter().sum::() as f64 / self.init_count as f64 + self.init_vals[..self.init_count].iter().sum::() as f64 + / self.init_count as f64 } _ => self.markers[2].1, } @@ -360,10 +361,7 @@ mod tests { est.update(v); } let p50 = est.estimate(); - assert!( - (400.0..=600.0).contains(&p50), - "p50={p50} not near 500" - ); + assert!((400.0..=600.0).contains(&p50), "p50={p50} not near 500"); } #[test] @@ -386,9 +384,6 @@ mod tests { est.update(v); } let p99 = est.estimate(); - assert!( - (480.0..=505.0).contains(&p99), - "p99={p99} not near 500" - ); + assert!((480.0..=505.0).contains(&p99), "p99={p99} not near 500"); } }