diff --git a/Cargo.lock b/Cargo.lock index 2b14716..fb5cb6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,6 +1829,7 @@ dependencies = [ "hex", "hmac", "prometheus", + "rand 0.8.5", "redis", "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index 94ac79f..528f33b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ anyhow = "1" axum = "0.7" governor = "0.6" prometheus = "0.13" +rand = "0.8" redis = { version = "0.25", features = ["aio", "tokio-comp", "connection-manager"] } reqwest = { version = "0.12", features = ["json"] } serde = { version = "1", features = ["derive"] } diff --git a/docs/STELLAR_RESILIENCE.md b/docs/STELLAR_RESILIENCE.md new file mode 100644 index 0000000..aac449d --- /dev/null +++ b/docs/STELLAR_RESILIENCE.md @@ -0,0 +1,118 @@ +# Stellar Client Circuit Breaker Recovery Runbook + +## Overview + +The Stellar client implements a circuit breaker pattern to protect against cascading failures when the Horizon API is degraded. This runbook documents the recovery procedure, state transitions, and operational metrics. + +## Circuit Breaker States + +| State | Description | Behavior | +|-------|-------------|----------| +| `Closed` | Normal operation | All requests pass through | +| `Open` | Failure threshold exceeded | All requests are rejected with `CircuitOpen` error | +| `HalfOpen` | Recovery probe phase | Limited concurrent requests allowed to test recovery | + +## State Transitions + +``` +Closed ──(failure_threshold exceeded)──► Open + ▲ │ + │ │ + └────(half_open_max_calls successes)────┘ + │ + ▼ + HalfOpen + │ + ├──(success)──► stays HalfOpen until max_calls reached + │ + └──(failure)──► Open +``` + +## Recovery Procedure + +### Automatic Recovery + +1. **Detection**: The circuit breaker trips to `Open` after `failure_threshold` consecutive failures. +2. **Wait**: The circuit stays `Open` for `open_duration` (default: 30s). +3. **Probe**: After `open_duration`, the circuit transitions to `HalfOpen`. +4. **Test**: Up to `half_open_max_calls` (default: 1) concurrent probe requests are allowed. +5. **Close**: If all probes succeed, the circuit returns to `Closed`. +6. **Reopen**: If any probe fails, the circuit returns to `Open` and the cycle repeats. + +### Manual Recovery + +If automatic recovery is not desired or is stuck: + +1. **Check Metrics**: Inspect `circuit_breaker_state` and `circuit_breaker_transitions_total` Prometheus metrics. +2. **Verify Horizon**: Confirm Horizon API health via `GET /health` or direct probe. +3. **Restart Service**: Restarting the service resets the circuit breaker to `Closed`. +4. **Hot Reload**: Use `POST /config/reload` to apply new circuit breaker settings without restart. + +## Configuration + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `STELLAR_CIRCUIT_BREAKER_FAILURE_THRESHOLD` | `5` | Failures before opening | +| `STELLAR_CIRCUIT_BREAKER_OPEN_DURATION_MS` | `30000` | Duration circuit stays open | +| `STELLAR_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS` | `1` | Max concurrent probes in half-open | +| `STELLAR_RETRY_JITTER_TYPE` | `full` | Jitter strategy: `none`, `full`, `equal`, `decorrelated` | +| `STELLAR_BULKHEAD_MAX_CONCURRENT` | `10` | Max concurrent Stellar requests | +| `STELLAR_BULKHEAD_MAX_QUEUE` | `100` | Max queued requests when bulkhead is full | + +## Metrics + +### Prometheus Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `circuit_breaker_state` | Gauge | Current state (0=closed, 1=open, 2=half_open) | +| `circuit_breaker_transitions_total{to_state}` | Counter | State transitions by target | +| `circuit_breaker_state_changes_total{from_state,to_state}` | Counter | State changes with source and target | +| `stellar_circuit_breaker_trips_total` | Counter | Times circuit opened | +| `stellar_circuit_breaker_recoveries_total` | Counter | Times circuit recovered to closed | +| `stellar_circuit_breaker_rejected_calls_total` | Counter | Calls rejected while open | +| `stellar_circuit_breaker_half_open_successes_total` | Counter | Successful half-open probes | +| `stellar_circuit_breaker_half_open_failures_total` | Counter | Failed half-open probes | +| `stellar_circuit_breaker_timeout_calls_total` | Counter | Timeout errors recorded | +| `stellar_circuit_breaker_retryable_http_calls_total` | Counter | Retryable HTTP errors recorded | + +### Programmatic Access + +```rust +let metrics = client.circuit_breaker_metrics(); +println!("trips: {}", metrics.trips); +println!("recoveries: {}", metrics.recoveries); +println!("half_open_successes: {}", metrics.half_open_successes); +println!("half_open_failures: {}", metrics.half_open_failures); +println!("rejected_calls: {}", metrics.rejected_calls); +println!("timeout_calls: {}", metrics.timeout_calls); +println!("retryable_http_calls: {}", metrics.retryable_http_calls); +``` + +## Chaos Testing + +The circuit breaker is designed to be chaos-monkey compatible: + +1. **Inject Failures**: Use a proxy to return 503 for a subset of requests. +2. **Verify Trip**: After `failure_threshold` failures, confirm circuit opens. +3. **Verify Rejection**: Confirm subsequent calls return `CircuitOpen` error. +4. **Verify Recovery**: After `open_duration`, confirm half-open probes succeed. +5. **Verify Close**: Confirm circuit returns to `Closed` after successful probes. + +## Graceful Degradation + +When the bulkhead is saturated: + +1. **Stale Cache Fallback**: If `graceful_degradation.stale_cache_ok` is enabled, return cached verification results. +2. **Network Error**: If no cache is available, return `VerificationStatus::NetworkError`. +3. **Metrics**: Bulkhead saturation is tracked via `rejected_calls` metric. + +## Troubleshooting + +| Symptom | Likely Cause | Resolution | +|---------|--------------|------------| +| Circuit stays open | Horizon is down | Wait for `open_duration` or restart | +| Frequent trips | Threshold too low | Increase `failure_threshold` | +| Slow recovery | `half_open_max_calls` too high | Reduce to 1 for faster probing | +| High latency | Retry backoff too aggressive | Reduce `base_delay` or disable jitter | +| Bulkhead saturated | Too many concurrent requests | Increase `max_concurrent` or add queue | diff --git a/src/cache.rs b/src/cache.rs index 90aeb0c..a2a2018 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -224,7 +224,7 @@ impl HealthCheckState { self.is_healthy = false; self.last_check = Some(SystemTime::now()); self.consecutive_failures += 1; - + // Exponential backoff: 2^failures seconds, capped at 60s let backoff_secs = (2u64.pow(self.consecutive_failures.min(6))).min(60); self.backoff_until = Some(SystemTime::now() + Duration::from_secs(backoff_secs)); @@ -249,12 +249,12 @@ impl RedisCache { async fn check_connection(&self) -> bool { let mut state = self.health_check_state.lock().await; - + // Return cached healthy status if we're not due for a check if !state.should_check() { return state.is_healthy; } - + // Perform health check let result = { let mut conn = self.connection.clone(); @@ -263,13 +263,13 @@ impl RedisCache { .await .is_ok() }; - + if result { state.record_success(); } else { state.record_failure(); } - + result } @@ -278,7 +278,7 @@ impl RedisCache { if !self.check_connection().await { return Err(anyhow::anyhow!("Redis connection is unhealthy")); } - + let mut conn = self.connection.clone(); let value: Option = conn.get(key).await?; Ok(value) @@ -289,7 +289,7 @@ impl RedisCache { if !self.check_connection().await { return Err(anyhow::anyhow!("Redis connection is unhealthy")); } - + let mut conn = self.connection.clone(); conn.set_ex::<_, _, ()>(key, value, ttl).await?; Ok(()) @@ -300,7 +300,7 @@ impl RedisCache { if !self.check_connection().await { return Err(anyhow::anyhow!("Redis connection is unhealthy")); } - + let mut conn = self.connection.clone(); conn.del::<_, ()>(key).await?; Ok(()) @@ -430,7 +430,7 @@ impl InMemoryCache { let mut store = self.store.write().await; let mut lru_queue = self.lru_queue.write().await; let mut stats = self.stats.write().await; - + let mut loaded = 0; for (key, value, ttl) in entries { store.insert( @@ -444,7 +444,7 @@ impl InMemoryCache { lru_queue.push_front(key); loaded += 1; } - + // Evict if over limit if self.max_size > 0 { while store.len() > self.max_size { @@ -456,7 +456,7 @@ impl InMemoryCache { } } } - + Ok(loaded) } @@ -473,14 +473,14 @@ impl InMemoryCache { current_size: store.len(), max_size: self.max_size, }; - + // Update Prometheus metrics if available if let Some(ref metrics) = self.metrics { metrics.set_cache_size(snapshot.current_size as u64); metrics.set_cache_hit_rate(snapshot.hit_rate); metrics.increment_cache_evictions_by(stats.evictions); } - + snapshot } @@ -489,20 +489,20 @@ impl InMemoryCache { let mut store = self.store.write().await; let mut lru_queue = self.lru_queue.write().await; let mut stats = self.stats.write().await; - + let mut results = Vec::with_capacity(keys.len()); let now = now_secs(); - + for key in keys { match store.get(key) { Some(entry) if entry.expires_at > now => { // Valid entry results.push((key.clone(), Some(entry.value.clone()))); - + // Update LRU lru_queue.retain(|k| k != key); lru_queue.push_front(key.clone()); - + stats.hits += 1; } Some(_) => { @@ -518,7 +518,7 @@ impl InMemoryCache { } } } - + Ok(results) } @@ -527,7 +527,7 @@ impl InMemoryCache { let mut store = self.store.write().await; let mut lru_queue = self.lru_queue.write().await; let mut stats = self.stats.write().await; - + for (key, value, ttl) in &entries { store.insert( key.clone(), @@ -539,7 +539,7 @@ impl InMemoryCache { lru_queue.retain(|k| k != key); lru_queue.push_front(key.clone()); } - + // Evict if over limit if self.max_size > 0 { while store.len() > self.max_size { @@ -551,7 +551,7 @@ impl InMemoryCache { } } } - + Ok(entries.len()) } @@ -571,16 +571,16 @@ impl InMemoryCache { let mut store = self.store.write().await; let mut lru_queue = self.lru_queue.write().await; let mut stats = self.stats.write().await; - + match store.get(key) { Some(entry) if entry.expires_at > now_secs() => { // Entry is valid - clone value while holding lock let value = entry.value.clone(); - + // Update LRU: move to front (most recently used) lru_queue.retain(|k| k != key); lru_queue.push_front(key.clone()); - + stats.hits += 1; Ok((Some(value), false)) } @@ -589,10 +589,10 @@ impl InMemoryCache { store.remove(key); lru_queue.retain(|k| k != key); stats.expired += 1; - + // Broadcast expiration event let _ = self.event_tx.send(CacheEvent::Expired { key: key.clone() }); - + Ok((None, true)) } None => { @@ -611,9 +611,9 @@ impl InMemoryCache { async fn set_raw(&self, key: &CacheKey, value: &str, ttl: u64) -> Result<()> { let mut store = self.store.write().await; let mut lru_queue = self.lru_queue.write().await; - - let is_update = store.contains_key(key); - + + let _is_update = store.contains_key(key); + // Insert or update entry store.insert( key.clone(), @@ -622,15 +622,15 @@ impl InMemoryCache { expires_at: now_secs().saturating_add(ttl), }, ); - + // Update LRU: move to front (most recently used) lru_queue.retain(|k| k != key); lru_queue.push_front(key.clone()); - + // Broadcast update event for any set (create or update) so subscribers // always receive the latest state change. let _ = self.event_tx.send(CacheEvent::Updated { key: key.clone() }); - + // Evict entries if over max_size if self.max_size > 0 { let mut stats = self.stats.write().await; @@ -638,15 +638,17 @@ impl InMemoryCache { if let Some(lru_key) = lru_queue.pop_back() { store.remove(&lru_key); stats.evictions += 1; - + // Broadcast eviction event - let _ = self.event_tx.send(CacheEvent::Evicted { key: lru_key.clone() }); + let _ = self.event_tx.send(CacheEvent::Evicted { + key: lru_key.clone(), + }); } else { break; } } } - + Ok(()) } @@ -655,11 +657,11 @@ impl InMemoryCache { let mut lru_queue = self.lru_queue.write().await; let existed = store.remove(key).is_some(); lru_queue.retain(|k| k != key); - + if existed { let _ = self.event_tx.send(CacheEvent::Deleted { key: key.clone() }); } - + Ok(()) } @@ -691,10 +693,9 @@ impl InMemoryCache { #[cfg(test)] mod tests { use super::*; - use futures::StreamExt; use std::time::Duration; - use tokio::time::sleep; use tokio::sync::broadcast::error::TryRecvError; + use tokio::time::sleep; #[tokio::test] async fn in_memory_cache_returns_value_within_ttl() { @@ -735,10 +736,7 @@ mod tests { let cache = CacheBackend::InMemory(InMemoryCache::new()); let v_key = CacheKey::Verification("same".to_string()); let c_key = CacheKey::Config("same".to_string()); - cache - .set_raw(&v_key, "verification_val", 60) - .await - .unwrap(); + cache.set_raw(&v_key, "verification_val", 60).await.unwrap(); cache.set_raw(&c_key, "config_val", 60).await.unwrap(); assert_eq!( cache.get_raw(&v_key).await.unwrap(), @@ -841,8 +839,14 @@ mod tests { cache.set_raw(&v_key, "verification_val", 60).await.unwrap(); cache.set_raw(&e_key, "events_val", 60).await.unwrap(); - assert_eq!(cache.get_raw(&v_key).await.unwrap(), Some("verification_val".to_string())); - assert_eq!(cache.get_raw(&e_key).await.unwrap(), Some("events_val".to_string())); + assert_eq!( + cache.get_raw(&v_key).await.unwrap(), + Some("verification_val".to_string()) + ); + assert_eq!( + cache.get_raw(&e_key).await.unwrap(), + Some("events_val".to_string()) + ); } #[tokio::test] @@ -878,7 +882,10 @@ mod tests { let cache_clone = Arc::clone(&cache); let key = CacheKey::Verification(format!("concurrent_write_{}", i)); handles.push(tokio::spawn(async move { - cache_clone.set_raw(&key, &format!("value_{}", i), 60).await.unwrap(); + cache_clone + .set_raw(&key, &format!("value_{}", i), 60) + .await + .unwrap(); cache_clone.get_raw(&key).await.unwrap() })); } @@ -901,7 +908,10 @@ mod tests { let backend_clone = Arc::clone(&backend); let key = CacheKey::Verification(format!("lru_{}", i)); handles.push(tokio::spawn(async move { - backend_clone.set_raw(&key, &format!("value_{}", i), 60).await.unwrap(); + backend_clone + .set_raw(&key, &format!("value_{}", i), 60) + .await + .unwrap(); })); } @@ -928,7 +938,14 @@ mod tests { // Concurrent hits // Ensure the key exists so these are hits, not misses. - backend.set_raw(&CacheKey::Verification("metric_concurrent".to_string()), "value", 60).await.unwrap(); + backend + .set_raw( + &CacheKey::Verification("metric_concurrent".to_string()), + "value", + 60, + ) + .await + .unwrap(); for _ in 0..50 { let backend_clone = Arc::clone(&backend); let key = CacheKey::Verification("metric_concurrent".to_string()); @@ -984,13 +1001,13 @@ mod tests { async fn redis_health_check_prevents_concurrent_checks() { // This test verifies the health check backoff mechanism let cache = RedisCache::new("redis://127.0.0.1:6379").await; - + // If Redis is not available, this will fail gracefully if cache.is_err() { // Skip test if Redis is not available return; } - + let cache = cache.unwrap(); let cache = Arc::new(cache); let mut handles = vec![]; @@ -998,9 +1015,9 @@ mod tests { // Trigger concurrent health checks for _ in 0..10 { let cache_clone = Arc::clone(&cache); - handles.push(tokio::spawn(async move { - cache_clone.check_connection().await - })); + handles.push(tokio::spawn( + async move { cache_clone.check_connection().await }, + )); } let results: Vec<_> = futures::future::join_all(handles).await; @@ -1014,31 +1031,67 @@ mod tests { async fn cache_warming_preloads_entries() { let cache = InMemoryCache::new(); let entries = vec![ - (CacheKey::Verification("warm1".to_string()), "value1".to_string(), 60), - (CacheKey::Verification("warm2".to_string()), "value2".to_string(), 60), - (CacheKey::Verification("warm3".to_string()), "value3".to_string(), 60), + ( + CacheKey::Verification("warm1".to_string()), + "value1".to_string(), + 60, + ), + ( + CacheKey::Verification("warm2".to_string()), + "value2".to_string(), + 60, + ), + ( + CacheKey::Verification("warm3".to_string()), + "value3".to_string(), + 60, + ), ]; - + let loaded = cache.warm(entries).await.unwrap(); assert_eq!(loaded, 3); - + // Verify entries are accessible let backend = CacheBackend::InMemory(cache); - assert_eq!(backend.get_raw(&CacheKey::Verification("warm1".to_string())).await.unwrap(), Some("value1".to_string())); - assert_eq!(backend.get_raw(&CacheKey::Verification("warm2".to_string())).await.unwrap(), Some("value2".to_string())); - assert_eq!(backend.get_raw(&CacheKey::Verification("warm3".to_string())).await.unwrap(), Some("value3".to_string())); + assert_eq!( + backend + .get_raw(&CacheKey::Verification("warm1".to_string())) + .await + .unwrap(), + Some("value1".to_string()) + ); + assert_eq!( + backend + .get_raw(&CacheKey::Verification("warm2".to_string())) + .await + .unwrap(), + Some("value2".to_string()) + ); + assert_eq!( + backend + .get_raw(&CacheKey::Verification("warm3".to_string())) + .await + .unwrap(), + Some("value3".to_string()) + ); } #[tokio::test] async fn cache_warming_respects_max_size() { let cache = InMemoryCache::with_max_size(5); - let entries: Vec<_> = (0..10).map(|i| { - (CacheKey::Verification(format!("warm_{}", i)), format!("value_{}", i), 60) - }).collect(); - + let entries: Vec<_> = (0..10) + .map(|i| { + ( + CacheKey::Verification(format!("warm_{}", i)), + format!("value_{}", i), + 60, + ) + }) + .collect(); + let loaded = cache.warm(entries).await.unwrap(); assert_eq!(loaded, 10); - + // Verify only 5 entries remain let stats = cache.stats().await; assert_eq!(stats.current_size, 5); @@ -1048,20 +1101,32 @@ mod tests { async fn cache_stats_track_operations() { let cache = InMemoryCache::new(); let backend = CacheBackend::InMemory(cache); - + // Miss - backend.get_raw(&CacheKey::Verification("miss".to_string())).await.unwrap(); - + backend + .get_raw(&CacheKey::Verification("miss".to_string())) + .await + .unwrap(); + // Set and hit - backend.set_raw(&CacheKey::Verification("hit".to_string()), "value", 60).await.unwrap(); - backend.get_raw(&CacheKey::Verification("hit".to_string())).await.unwrap(); - backend.get_raw(&CacheKey::Verification("hit".to_string())).await.unwrap(); - + backend + .set_raw(&CacheKey::Verification("hit".to_string()), "value", 60) + .await + .unwrap(); + backend + .get_raw(&CacheKey::Verification("hit".to_string())) + .await + .unwrap(); + backend + .get_raw(&CacheKey::Verification("hit".to_string())) + .await + .unwrap(); + let cache = match backend { CacheBackend::InMemory(c) => c, _ => std::panic!("Expected InMemoryCache"), }; - + let stats = cache.stats().await; assert_eq!(stats.hits, 2); assert_eq!(stats.misses, 1); @@ -1072,17 +1137,24 @@ mod tests { async fn cache_stats_track_evictions() { let cache = InMemoryCache::with_max_size(3); let backend = CacheBackend::InMemory(cache); - + // Add 5 entries (should evict 2) for i in 0..5 { - backend.set_raw(&CacheKey::Verification(format!("evict_{}", i)), &format!("value_{}", i), 60).await.unwrap(); + backend + .set_raw( + &CacheKey::Verification(format!("evict_{}", i)), + &format!("value_{}", i), + 60, + ) + .await + .unwrap(); } - + let cache = match backend { CacheBackend::InMemory(c) => c, _ => std::panic!("Expected InMemoryCache"), }; - + let stats = cache.stats().await; assert_eq!(stats.evictions, 2); assert_eq!(stats.current_size, 3); @@ -1092,23 +1164,32 @@ mod tests { async fn batch_get_retrieves_multiple_keys() { let cache = InMemoryCache::new(); let backend = CacheBackend::InMemory(cache); - - backend.set_raw(&CacheKey::Verification("key1".to_string()), "value1", 60).await.unwrap(); - backend.set_raw(&CacheKey::Verification("key2".to_string()), "value2", 60).await.unwrap(); - backend.set_raw(&CacheKey::Verification("key3".to_string()), "value3", 60).await.unwrap(); - + + backend + .set_raw(&CacheKey::Verification("key1".to_string()), "value1", 60) + .await + .unwrap(); + backend + .set_raw(&CacheKey::Verification("key2".to_string()), "value2", 60) + .await + .unwrap(); + backend + .set_raw(&CacheKey::Verification("key3".to_string()), "value3", 60) + .await + .unwrap(); + let cache = match backend { CacheBackend::InMemory(c) => c, _ => std::panic!("Expected InMemoryCache"), }; - + let keys = vec![ CacheKey::Verification("key1".to_string()), CacheKey::Verification("key2".to_string()), CacheKey::Verification("key3".to_string()), CacheKey::Verification("key4".to_string()), // miss ]; - + let results = cache.get_batch(&keys).await.unwrap(); assert_eq!(results.len(), 4); assert_eq!(results[0].1, Some("value1".to_string())); @@ -1121,14 +1202,26 @@ mod tests { async fn batch_set_stores_multiple_keys() { let cache = InMemoryCache::new(); let entries = vec![ - (CacheKey::Verification("batch1".to_string()), "value1".to_string(), 60), - (CacheKey::Verification("batch2".to_string()), "value2".to_string(), 60), - (CacheKey::Verification("batch3".to_string()), "value3".to_string(), 60), + ( + CacheKey::Verification("batch1".to_string()), + "value1".to_string(), + 60, + ), + ( + CacheKey::Verification("batch2".to_string()), + "value2".to_string(), + 60, + ), + ( + CacheKey::Verification("batch3".to_string()), + "value3".to_string(), + 60, + ), ]; - + let count = cache.set_batch(entries).await.unwrap(); assert_eq!(count, 3); - + let stats = cache.stats().await; assert_eq!(stats.current_size, 3); } @@ -1137,25 +1230,42 @@ mod tests { async fn event_driven_invalidation_broadcasts_events() { let cache = InMemoryCache::new(); let mut rx = cache.subscribe(); - + // Set a key - cache.set_raw(&CacheKey::Verification("event_key".to_string()), "value", 60).await.unwrap(); - + cache + .set_raw( + &CacheKey::Verification("event_key".to_string()), + "value", + 60, + ) + .await + .unwrap(); + // Update it - cache.set_raw(&CacheKey::Verification("event_key".to_string()), "new_value", 60).await.unwrap(); - + cache + .set_raw( + &CacheKey::Verification("event_key".to_string()), + "new_value", + 60, + ) + .await + .unwrap(); + // Delete it - cache.delete(&CacheKey::Verification("event_key".to_string())).await.unwrap(); - + cache + .delete(&CacheKey::Verification("event_key".to_string())) + .await + .unwrap(); + // Check events with timeout let event1 = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await; assert!(event1.is_ok()); matches!(event1.unwrap().unwrap(), CacheEvent::Updated { .. }); - + let event2 = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await; assert!(event2.is_ok()); matches!(event2.unwrap().unwrap(), CacheEvent::Updated { .. }); - + let event3 = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await; assert!(event3.is_ok()); matches!(event3.unwrap().unwrap(), CacheEvent::Deleted { .. }); @@ -1165,8 +1275,15 @@ mod tests { async fn event_broadcasts_on_expiry() { let cache = InMemoryCache::new(); let mut rx = cache.subscribe(); - - cache.set_raw(&CacheKey::Verification("expire_key".to_string()), "value", 1).await.unwrap(); + + cache + .set_raw( + &CacheKey::Verification("expire_key".to_string()), + "value", + 1, + ) + .await + .unwrap(); // Drain any prior events (e.g., initial Updated from set) loop { match rx.try_recv() { @@ -1179,7 +1296,10 @@ mod tests { sleep(Duration::from_secs(2)).await; // Trigger expiry check - cache.get_raw(&CacheKey::Verification("expire_key".to_string())).await.unwrap(); + cache + .get_raw(&CacheKey::Verification("expire_key".to_string())) + .await + .unwrap(); // Check for expiration event let event = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await; @@ -1196,9 +1316,18 @@ mod tests { let cache = InMemoryCache::with_max_size(2); let mut rx = cache.subscribe(); // Perform sets that should trigger eviction - cache.set_raw(&CacheKey::Verification("evict1".to_string()), "value1", 60).await.unwrap(); - cache.set_raw(&CacheKey::Verification("evict2".to_string()), "value2", 60).await.unwrap(); - cache.set_raw(&CacheKey::Verification("evict3".to_string()), "value3", 60).await.unwrap(); + cache + .set_raw(&CacheKey::Verification("evict1".to_string()), "value1", 60) + .await + .unwrap(); + cache + .set_raw(&CacheKey::Verification("evict2".to_string()), "value2", 60) + .await + .unwrap(); + cache + .set_raw(&CacheKey::Verification("evict3".to_string()), "value3", 60) + .await + .unwrap(); // Consume events until we see an Evicted event or timeout let deadline = std::time::Instant::now() + Duration::from_millis(500); @@ -1222,14 +1351,20 @@ mod tests { let cache = InMemoryCache::new(); let mut rx1 = cache.subscribe(); let mut rx2 = cache.subscribe(); - - cache.set_raw(&CacheKey::Verification("multi".to_string()), "value", 60).await.unwrap(); - cache.delete(&CacheKey::Verification("multi".to_string())).await.unwrap(); - + + cache + .set_raw(&CacheKey::Verification("multi".to_string()), "value", 60) + .await + .unwrap(); + cache + .delete(&CacheKey::Verification("multi".to_string())) + .await + .unwrap(); + // Both subscribers should receive events let event1 = tokio::time::timeout(Duration::from_millis(100), rx1.recv()).await; let event2 = tokio::time::timeout(Duration::from_millis(100), rx2.recv()).await; - + assert!(event1.is_ok()); assert!(event2.is_ok()); } diff --git a/src/config.rs b/src/config.rs index e82c8c1..b0cea27 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,11 @@ -use std::{env, fmt, string::{String, ToString}, sync::Arc, vec::Vec}; -use thiserror::Error; +use std::{ + env, fmt, + string::{String, ToString}, + sync::Arc, + vec::Vec, +}; use stellar_strkey::ed25519::PrivateKey; +use thiserror::Error; use crate::metrics::MetricsRegistry; @@ -12,6 +17,9 @@ const DEFAULT_STELLAR_REQUEST_TIMEOUT_MS: u64 = 10_000; const DEFAULT_STELLAR_CIRCUIT_BREAKER_FAILURE_THRESHOLD: u32 = 5; const DEFAULT_STELLAR_CIRCUIT_BREAKER_OPEN_DURATION_MS: u64 = 30_000; const DEFAULT_STELLAR_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS: u32 = 1; +const DEFAULT_STELLAR_RETRY_JITTER_TYPE: &str = "full"; +const DEFAULT_STELLAR_BULKHEAD_MAX_CONCURRENT: u32 = 10; +const DEFAULT_STELLAR_BULKHEAD_MAX_QUEUE: u32 = 100; /// Current configuration schema version. /// Increment this when adding or removing fields that break backward compatibility. @@ -38,9 +46,8 @@ pub struct ValidatedUrl(url::Url); impl ValidatedUrl { pub fn parse(input: &str) -> Result { - let url = url::Url::parse(input).map_err(|_| { - ConfigError::Validation(format!("invalid URL: '{}'", input)) - })?; + let url = url::Url::parse(input) + .map_err(|_| ConfigError::Validation(format!("invalid URL: '{}'", input)))?; if url.scheme() != "http" && url.scheme() != "https" { return Err(ConfigError::Validation(format!( "URL must use http or https scheme, got '{}' in '{}'", @@ -72,9 +79,8 @@ pub struct ValidatedRedisUrl(url::Url); impl ValidatedRedisUrl { pub fn parse(input: &str) -> Result { - let url = url::Url::parse(input).map_err(|_| { - ConfigError::Validation(format!("invalid Redis URL: '{}'", input)) - })?; + let url = url::Url::parse(input) + .map_err(|_| ConfigError::Validation(format!("invalid Redis URL: '{}'", input)))?; if url.scheme() != "redis" && url.scheme() != "rediss" { return Err(ConfigError::Validation(format!( "REDIS_URL must use redis:// or rediss:// scheme, got '{}'", @@ -176,7 +182,9 @@ impl ConfigUpdate { pub type ConfigWatcher = std::sync::Arc>; /// Create a new hot-reload channel with the given initial config. -pub fn config_channel(initial: AppConfig) -> Result<(ConfigWatcher, tokio::sync::watch::Receiver), ConfigError> { +pub fn config_channel( + initial: AppConfig, +) -> Result<(ConfigWatcher, tokio::sync::watch::Receiver), ConfigError> { let update = ConfigUpdate::new(initial)?; let (tx, rx) = tokio::sync::watch::channel(update); Ok((std::sync::Arc::new(tx), rx)) @@ -221,6 +229,9 @@ pub struct AppConfig { pub stellar_circuit_breaker_failure_threshold: u32, pub stellar_circuit_breaker_open_duration_ms: u64, pub stellar_circuit_breaker_half_open_max_calls: u32, + pub stellar_retry_jitter_type: String, + pub stellar_bulkhead_max_concurrent: u32, + pub stellar_bulkhead_max_queue: u32, pub log_level: String, pub webhook_urls: Vec, pub webhook_secret: Option, @@ -250,20 +261,35 @@ impl fmt::Debug for AppConfig { .field("redis_url", &self.redis_url) .field("rate_limit_per_second", &self.rate_limit_per_second) .field("rate_limit_burst", &self.rate_limit_burst) - .field("per_issuer_rate_limit_per_second", &self.per_issuer_rate_limit_per_second) - .field("per_issuer_rate_limit_burst", &self.per_issuer_rate_limit_burst) - .field("issuer_rate_limit_ttl_seconds", &self.issuer_rate_limit_ttl_seconds) + .field( + "per_issuer_rate_limit_per_second", + &self.per_issuer_rate_limit_per_second, + ) + .field( + "per_issuer_rate_limit_burst", + &self.per_issuer_rate_limit_burst, + ) + .field( + "issuer_rate_limit_ttl_seconds", + &self.issuer_rate_limit_ttl_seconds, + ) .field("stellar_max_retries", &self.stellar_max_retries) .field( "stellar_retry_base_delay_ms", &self.stellar_retry_base_delay_ms, ) - .field("stellar_retry_max_delay_ms", &self.stellar_retry_max_delay_ms) + .field( + "stellar_retry_max_delay_ms", + &self.stellar_retry_max_delay_ms, + ) .field( "stellar_retry_jitter_enabled", &self.stellar_retry_jitter_enabled, ) - .field("stellar_request_timeout_ms", &self.stellar_request_timeout_ms) + .field( + "stellar_request_timeout_ms", + &self.stellar_request_timeout_ms, + ) .field( "stellar_circuit_breaker_failure_threshold", &self.stellar_circuit_breaker_failure_threshold, @@ -276,6 +302,15 @@ impl fmt::Debug for AppConfig { "stellar_circuit_breaker_half_open_max_calls", &self.stellar_circuit_breaker_half_open_max_calls, ) + .field("stellar_retry_jitter_type", &self.stellar_retry_jitter_type) + .field( + "stellar_bulkhead_max_concurrent", + &self.stellar_bulkhead_max_concurrent, + ) + .field( + "stellar_bulkhead_max_queue", + &self.stellar_bulkhead_max_queue, + ) .field("log_level", &self.log_level) .field("webhook_urls", &self.webhook_urls) .field( @@ -283,9 +318,18 @@ impl fmt::Debug for AppConfig { &self.webhook_secret.as_deref().map(|_| ""), ) .field("webhook_max_retries", &self.webhook_max_retries) - .field("webhook_retry_base_delay_ms", &self.webhook_retry_base_delay_ms) - .field("webhook_retry_max_delay_ms", &self.webhook_retry_max_delay_ms) - .field("webhook_request_timeout_ms", &self.webhook_request_timeout_ms) + .field( + "webhook_retry_base_delay_ms", + &self.webhook_retry_base_delay_ms, + ) + .field( + "webhook_retry_max_delay_ms", + &self.webhook_retry_max_delay_ms, + ) + .field( + "webhook_request_timeout_ms", + &self.webhook_request_timeout_ms, + ) .field("webhook_jitter_enabled", &self.webhook_jitter_enabled) .field("cache_verification_ttl", &self.cache_verification_ttl) .field("cache_backend", &self.cache_backend) @@ -322,7 +366,7 @@ impl AppConfig { // ── Environment variable documentation ────────────────────────── // Each variable is documented with its purpose, default, and validation rules. - // + // // NETWORK: // PORT - HTTP listen port (1-65535, default: 8080) // LOG_LEVEL - Logging level (default: "info") @@ -376,8 +420,7 @@ impl AppConfig { Ok(key) => { if PrivateKey::from_string(&key).is_err() { errors.push( - "STELLAR_SECRET_KEY must be a valid Stellar ed25519 secret key" - .to_string(), + "STELLAR_SECRET_KEY must be a valid Stellar ed25519 secret key".to_string(), ); } Some(key) @@ -443,6 +486,18 @@ impl AppConfig { "STELLAR_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS", &DEFAULT_STELLAR_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS.to_string(), ); + let stellar_retry_jitter_type_raw = get_env_or_default( + "STELLAR_RETRY_JITTER_TYPE", + DEFAULT_STELLAR_RETRY_JITTER_TYPE, + ); + let stellar_bulkhead_max_concurrent_raw = get_env_or_default( + "STELLAR_BULKHEAD_MAX_CONCURRENT", + &DEFAULT_STELLAR_BULKHEAD_MAX_CONCURRENT.to_string(), + ); + let stellar_bulkhead_max_queue_raw = get_env_or_default( + "STELLAR_BULKHEAD_MAX_QUEUE", + &DEFAULT_STELLAR_BULKHEAD_MAX_QUEUE.to_string(), + ); let cache_verification_ttl_raw = get_env_or_default("CACHE_VERIFICATION_TTL", "3600"); let cache_backend_raw = get_env_or_default("CACHE_BACKEND", "inmemory"); let cache_max_size_raw = get_env_or_default("CACHE_MAX_SIZE", "10000"); @@ -459,7 +514,10 @@ impl AppConfig { } }, Err(_) => { - errors.push(format!("PORT must be a valid u16 (1-65535), got '{}'", port_raw)); + errors.push(format!( + "PORT must be a valid u16 (1-65535), got '{}'", + port_raw + )); 8080 } }; @@ -521,9 +579,7 @@ impl AppConfig { let per_issuer_rate_limit_per_second: u32 = match per_issuer_rps_raw.parse() { Ok(v) if v > 0 => v, Ok(_) => { - errors.push( - "PER_ISSUER_RATE_LIMIT_PER_SECOND must be greater than 0".to_string(), - ); + errors.push("PER_ISSUER_RATE_LIMIT_PER_SECOND must be greater than 0".to_string()); 10 } Err(_) => { @@ -538,9 +594,7 @@ impl AppConfig { let per_issuer_rate_limit_burst: u32 = match per_issuer_burst_raw.parse() { Ok(v) if v > 0 => v, Ok(_) => { - errors.push( - "PER_ISSUER_RATE_LIMIT_BURST must be greater than 0".to_string(), - ); + errors.push("PER_ISSUER_RATE_LIMIT_BURST must be greater than 0".to_string()); per_issuer_rate_limit_per_second * 2 } Err(_) => { @@ -570,9 +624,7 @@ impl AppConfig { let issuer_rate_limit_ttl_seconds: u64 = match issuer_ttl_raw.parse() { Ok(v) if v > 0 => v, Ok(_) => { - errors.push( - "ISSUER_RATE_LIMIT_TTL_SECONDS must be greater than 0".to_string(), - ); + errors.push("ISSUER_RATE_LIMIT_TTL_SECONDS must be greater than 0".to_string()); 3600 } Err(_) => { @@ -737,6 +789,48 @@ impl AppConfig { DEFAULT_STELLAR_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS } }; + let stellar_retry_jitter_type = match stellar_retry_jitter_type_raw.to_lowercase().as_str() + { + "none" | "full" | "equal" | "decorrelated" => stellar_retry_jitter_type_raw.to_string(), + other => { + errors.push(format!( + "STELLAR_RETRY_JITTER_TYPE must be one of: none, full, equal, decorrelated; got '{}'", + other + )); + DEFAULT_STELLAR_RETRY_JITTER_TYPE.to_string() + } + }; + + let stellar_bulkhead_max_concurrent: u32 = match stellar_bulkhead_max_concurrent_raw.parse() + { + Ok(v) if v > 0 => v, + Ok(_) => { + errors.push("STELLAR_BULKHEAD_MAX_CONCURRENT must be greater than 0".to_string()); + DEFAULT_STELLAR_BULKHEAD_MAX_CONCURRENT + } + Err(_) => { + errors.push(format!( + "STELLAR_BULKHEAD_MAX_CONCURRENT must be a valid u32, got '{}'", + stellar_bulkhead_max_concurrent_raw + )); + DEFAULT_STELLAR_BULKHEAD_MAX_CONCURRENT + } + }; + + let stellar_bulkhead_max_queue: u32 = match stellar_bulkhead_max_queue_raw.parse() { + Ok(v) if v > 0 => v, + Ok(_) => { + errors.push("STELLAR_BULKHEAD_MAX_QUEUE must be greater than 0".to_string()); + DEFAULT_STELLAR_BULKHEAD_MAX_QUEUE + } + Err(_) => { + errors.push(format!( + "STELLAR_BULKHEAD_MAX_QUEUE must be a valid u32, got '{}'", + stellar_bulkhead_max_queue_raw + )); + DEFAULT_STELLAR_BULKHEAD_MAX_QUEUE + } + }; let cache_verification_ttl: u64 = match cache_verification_ttl_raw.parse() { Ok(v) => v, @@ -914,7 +1008,10 @@ impl AppConfig { .filter(|s| !s.is_empty()) .map(|url| { if ValidatedUrl::parse(url).is_err() { - errors.push(format!("WEBHOOK_URLS must contain valid URLs, got '{}'", url)); + errors.push(format!( + "WEBHOOK_URLS must contain valid URLs, got '{}'", + url + )); } url.to_string() }) @@ -951,6 +1048,9 @@ impl AppConfig { stellar_circuit_breaker_failure_threshold, stellar_circuit_breaker_open_duration_ms, stellar_circuit_breaker_half_open_max_calls, + stellar_retry_jitter_type, + stellar_bulkhead_max_concurrent, + stellar_bulkhead_max_queue, log_level, webhook_urls, webhook_secret, @@ -969,7 +1069,10 @@ impl AppConfig { /// Reload configuration from environment variables, returning a new instance. /// Useful in combination with the hot-reload mechanism to apply changes at runtime. - pub fn reload_from_env(&self, metrics: Option>) -> Result { + pub fn reload_from_env( + &self, + metrics: Option>, + ) -> Result { Self::from_env_with_metrics(metrics) } @@ -1016,6 +1119,9 @@ mod tests { "STELLAR_CIRCUIT_BREAKER_FAILURE_THRESHOLD", "STELLAR_CIRCUIT_BREAKER_OPEN_DURATION_MS", "STELLAR_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS", + "STELLAR_RETRY_JITTER_TYPE", + "STELLAR_BULKHEAD_MAX_CONCURRENT", + "STELLAR_BULKHEAD_MAX_QUEUE", "LOG_LEVEL", "WEBHOOK_URLS", "WEBHOOK_SECRET", @@ -1045,7 +1151,10 @@ mod tests { let cfg = AppConfig::from_env().expect("config should load with defaults"); assert_eq!(cfg.port, 8080); - assert_eq!(cfg.stellar_horizon_url, "https://horizon-testnet.stellar.org"); + assert_eq!( + cfg.stellar_horizon_url, + "https://horizon-testnet.stellar.org" + ); assert_eq!(cfg.redis_url, "redis://127.0.0.1:6379"); assert_eq!(cfg.rate_limit_per_second, 100); assert_eq!(cfg.per_issuer_rate_limit_per_second, 10); @@ -1060,6 +1169,9 @@ mod tests { assert_eq!(cfg.stellar_circuit_breaker_failure_threshold, 5); assert_eq!(cfg.stellar_circuit_breaker_open_duration_ms, 30_000); assert_eq!(cfg.stellar_circuit_breaker_half_open_max_calls, 1); + assert_eq!(cfg.stellar_retry_jitter_type, "full"); + assert_eq!(cfg.stellar_bulkhead_max_concurrent, 10); + assert_eq!(cfg.stellar_bulkhead_max_queue, 100); } #[test] @@ -1112,9 +1224,17 @@ mod tests { let err = AppConfig::from_env().expect_err("config should fail"); let msg = err.to_string(); - assert!(msg.contains("port 0 is below minimum 1") || msg.contains("PORT must be a valid u16")); - assert!(msg.contains("STELLAR_HORIZON_URL must be a valid http/https URL") || msg.contains("invalid URL")); - assert!(msg.contains("REDIS_URL must be a valid redis:// or rediss:// URL") || msg.contains("invalid Redis URL")); + assert!( + msg.contains("port 0 is below minimum 1") || msg.contains("PORT must be a valid u16") + ); + assert!( + msg.contains("STELLAR_HORIZON_URL must be a valid http/https URL") + || msg.contains("invalid URL") + ); + assert!( + msg.contains("REDIS_URL must be a valid redis:// or rediss:// URL") + || msg.contains("invalid Redis URL") + ); assert!(msg.contains("RATE_LIMIT_PER_SECOND must be greater than 0")); assert!(msg.contains("RATE_LIMIT_BURST must be greater than 0")); assert!(msg.contains("RATE_LIMIT_BURST") && msg.contains("must be >=")); @@ -1203,6 +1323,9 @@ mod tests { stellar_circuit_breaker_failure_threshold: 5, stellar_circuit_breaker_open_duration_ms: 30_000, stellar_circuit_breaker_half_open_max_calls: 1, + stellar_retry_jitter_type: "full".to_string(), + stellar_bulkhead_max_concurrent: 10, + stellar_bulkhead_max_queue: 100, log_level: "info".to_string(), webhook_urls: vec!["https://webhook.example.com".to_string()], webhook_secret: Some("another-secret".to_string()), @@ -1246,8 +1369,8 @@ mod tests { env::set_var("STELLAR_SECRET_KEY", VALID_KEY); let metrics = MetricsRegistry::arc(); - let _cfg = AppConfig::from_env_with_metrics(Some(Arc::clone(&metrics))) - .expect("should succeed"); + let _cfg = + AppConfig::from_env_with_metrics(Some(Arc::clone(&metrics))).expect("should succeed"); let output = metrics.render(); assert!(output.contains("config_reload_total")); @@ -1264,7 +1387,9 @@ mod tests { env::set_var("RATE_LIMIT_BURST", "50"); // burst < rps let err = AppConfig::from_env().expect_err("should fail"); - assert!(err.to_string().contains("RATE_LIMIT_BURST (50) must be >= RATE_LIMIT_PER_SECOND (100)")); + assert!(err + .to_string() + .contains("RATE_LIMIT_BURST (50) must be >= RATE_LIMIT_PER_SECOND (100)")); } #[test] @@ -1277,7 +1402,9 @@ mod tests { env::set_var("PER_ISSUER_RATE_LIMIT_BURST", "10"); // burst < rps let err = AppConfig::from_env().expect_err("should fail"); - assert!(err.to_string().contains("PER_ISSUER_RATE_LIMIT_BURST (10) must be >= PER_ISSUER_RATE_LIMIT_PER_SECOND (20)")); + assert!(err.to_string().contains( + "PER_ISSUER_RATE_LIMIT_BURST (10) must be >= PER_ISSUER_RATE_LIMIT_PER_SECOND (20)" + )); } #[test] @@ -1393,4 +1520,4 @@ mod tests { fn config_version_constant_is_consistent() { assert_eq!(CONFIG_VERSION, 1); } -} \ No newline at end of file +} diff --git a/src/event.rs b/src/event.rs index adb1197..f29a3a6 100644 --- a/src/event.rs +++ b/src/event.rs @@ -62,11 +62,7 @@ impl ContractEventContext { pub fn idempotency_key(&self, aggregate_id: &str, event_type: &str) -> String { format!( "contract:{}:{}:{}:{}:{}", - self.transaction_hash, - self.ledger_sequence, - self.event_index, - aggregate_id, - event_type + self.transaction_hash, self.ledger_sequence, self.event_index, aggregate_id, event_type ) } @@ -235,10 +231,7 @@ impl EventIngestor { } } - pub fn with_metrics( - mut self, - metrics: Arc, - ) -> Self { + pub fn with_metrics(mut self, metrics: Arc) -> Self { self.metrics = Some(metrics); self } @@ -484,9 +477,7 @@ mod tests { ) .expect_err("context should fail validation"); - assert!(err - .to_string() - .contains("invalid contract event context")); + assert!(err.to_string().contains("invalid contract event context")); } #[test] @@ -678,7 +669,10 @@ mod store_tests { assert_eq!(store.count("doc-1"), 1); assert_eq!(store.count("doc-2"), 1); - assert_eq!(store.get_history("doc-1").unwrap()[0].event_type, EVENT_DOCUMENT_REGISTERED); + assert_eq!( + store.get_history("doc-1").unwrap()[0].event_type, + EVENT_DOCUMENT_REGISTERED + ); assert_eq!( store.get_history("doc-2").unwrap()[0].event_type, EVENT_DOCUMENT_OWNER_CHANGED diff --git a/src/hash_validator.rs b/src/hash_validator.rs index dfb6846..51a8315 100644 --- a/src/hash_validator.rs +++ b/src/hash_validator.rs @@ -55,8 +55,14 @@ pub const STELLAR_MEMO_HASH_BYTES: usize = 32; /// completed; see [`HashValidator::validate_with_length_constant_time`]. #[derive(Debug, PartialEq, Eq)] pub enum ValidationError { - WrongLength { expected: usize, actual: usize }, - InvalidCharacter { position: usize, character: char }, + WrongLength { + expected: usize, + actual: usize, + }, + InvalidCharacter { + position: usize, + character: char, + }, EmptyHash, /// Hash algorithm is not supported for contract submission (only SHA-256 is accepted). UnsupportedAlgorithm, @@ -72,17 +78,31 @@ impl fmt::Display for ValidationError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { ValidationError::WrongLength { expected, actual } => { - write!(f, "hash length {} does not match expected {}", actual, expected) + write!( + f, + "hash length {} does not match expected {}", + actual, expected + ) } - ValidationError::InvalidCharacter { position, character } => { - write!(f, "invalid character '{}' at position {}", character, position) + ValidationError::InvalidCharacter { + position, + character, + } => { + write!( + f, + "invalid character '{}' at position {}", + character, position + ) } ValidationError::EmptyHash => write!(f, "hash cannot be empty"), ValidationError::UnsupportedAlgorithm => { write!(f, "hash algorithm not supported for contract submission") } ValidationError::NotCanonical => { - write!(f, "hash is not in canonical form (must be lowercase hex without whitespace)") + write!( + f, + "hash is not in canonical form (must be lowercase hex without whitespace)" + ) } ValidationError::InvalidStellarMemoFormat => { write!(f, "hash does not match Stellar memo format requirements") @@ -161,9 +181,7 @@ impl CanonicalHash { // whitespace issue; any other length composed of valid hex is a length // error. if !is_canonical_shape(hash) { - let all_hex = hash - .bytes() - .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); + let all_hex = hash.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); if all_hex { return Err(ValidationError::WrongLength { expected: HashAlgorithm::SHA256.hex_length(), @@ -300,7 +318,12 @@ impl HashRegistry { // on the first match. let mut seen = false; for existing in self.inner.iter() { - if existing.key().as_bytes().ct_eq(hash.as_str().as_bytes()).into() { + if existing + .key() + .as_bytes() + .ct_eq(hash.as_str().as_bytes()) + .into() + { seen = true; } } @@ -318,7 +341,12 @@ impl HashRegistry { pub fn contains(&self, hash: &CanonicalHash) -> bool { let mut found = false; for existing in self.inner.iter() { - if existing.key().as_bytes().ct_eq(hash.as_str().as_bytes()).into() { + if existing + .key() + .as_bytes() + .ct_eq(hash.as_str().as_bytes()) + .into() + { found = true; } } @@ -595,7 +623,9 @@ fn is_canonical_shape_with_len(hash: &str, len: usize) -> bool { if bytes.len() != len { return false; } - bytes.iter().all(|&b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + bytes + .iter() + .all(|&b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) } /// Canonical shape for the contract algorithm (SHA-256, 64 chars). @@ -853,7 +883,11 @@ mod tests { #[test] fn canonical_hash_from_bytes() { - let bytes = [0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55]; + let bytes = [ + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, + 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, + 0x78, 0x52, 0xb8, 0x55, + ]; let hash = CanonicalHash::from_bytes(&bytes); assert_eq!(hash.as_str(), sample_sha256()); } @@ -993,7 +1027,10 @@ mod tests { #[test] fn is_canonical_detects_valid_hash() { - assert!(HashValidator::is_canonical(sample_sha256(), HashAlgorithm::SHA256)); + assert!(HashValidator::is_canonical( + sample_sha256(), + HashAlgorithm::SHA256 + )); } #[test] @@ -1004,7 +1041,10 @@ mod tests { #[test] fn is_canonical_rejects_wrong_length() { - assert!(!HashValidator::is_canonical("abc123", HashAlgorithm::SHA256)); + assert!(!HashValidator::is_canonical( + "abc123", + HashAlgorithm::SHA256 + )); } // ── HashAlgorithm methods ─────────────────────────────────────────── @@ -1025,7 +1065,10 @@ mod tests { #[test] fn validation_error_display_wrong_length() { - let err = ValidationError::WrongLength { expected: 64, actual: 63 }; + let err = ValidationError::WrongLength { + expected: 64, + actual: 63, + }; let msg = format!("{}", err); assert!(msg.contains("63")); assert!(msg.contains("64")); @@ -1033,7 +1076,10 @@ mod tests { #[test] fn validation_error_display_invalid_character() { - let err = ValidationError::InvalidCharacter { position: 10, character: 'g' }; + let err = ValidationError::InvalidCharacter { + position: 10, + character: 'g', + }; let msg = format!("{}", err); assert!(msg.contains("10")); assert!(msg.contains("g")); diff --git a/src/lib.rs b/src/lib.rs index e1b0d59..7a6c65d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -476,7 +476,9 @@ impl ProofStellContract { /// Returns the stored admin address, or `None` if the contract is not yet initialized. pub fn get_admin(env: Env) -> Option
{ - env.storage().persistent().get::(&DataKey::Admin) + env.storage() + .persistent() + .get::(&DataKey::Admin) } /// Upgrades the contract WASM to the given hash. @@ -785,9 +787,18 @@ mod tests { let docs = soroban_sdk::vec![ &env, - DocumentInfo { owner: owner.clone(), document_hash: BytesN::from_array(&env, &[1; 32]) }, - DocumentInfo { owner: owner.clone(), document_hash: BytesN::from_array(&env, &[2; 32]) }, - DocumentInfo { owner: owner.clone(), document_hash: BytesN::from_array(&env, &[3; 32]) }, + DocumentInfo { + owner: owner.clone(), + document_hash: BytesN::from_array(&env, &[1; 32]) + }, + DocumentInfo { + owner: owner.clone(), + document_hash: BytesN::from_array(&env, &[2; 32]) + }, + DocumentInfo { + owner: owner.clone(), + document_hash: BytesN::from_array(&env, &[3; 32]) + }, ]; let records = client.batch_register_documents(&issuer, &docs); @@ -809,9 +820,18 @@ mod tests { let docs = soroban_sdk::vec![ &env, - DocumentInfo { owner: owner.clone(), document_hash: BytesN::from_array(&env, &[3; 32]) }, - DocumentInfo { owner: owner.clone(), document_hash: hash1.clone() }, - DocumentInfo { owner: owner.clone(), document_hash: hash2.clone() }, + DocumentInfo { + owner: owner.clone(), + document_hash: BytesN::from_array(&env, &[3; 32]) + }, + DocumentInfo { + owner: owner.clone(), + document_hash: hash1.clone() + }, + DocumentInfo { + owner: owner.clone(), + document_hash: hash2.clone() + }, ]; let err = client @@ -872,7 +892,12 @@ mod tests { client.register_document(&issuer, &owner, h); } - let hash_vec = soroban_sdk::vec![&env, hashes[0].clone(), hashes[1].clone(), hashes[2].clone()]; + let hash_vec = soroban_sdk::vec![ + &env, + hashes[0].clone(), + hashes[1].clone(), + hashes[2].clone() + ]; let records = client.batch_revoke_documents(&issuer, &hash_vec); assert_eq!(records.len(), 3); @@ -1016,10 +1041,7 @@ mod tests { let admin = Address::generate(&env); let wasm_hash = BytesN::from_array(&env, &[0u8; 32]); - let err = client - .try_upgrade(&admin, &wasm_hash) - .unwrap_err() - .unwrap(); + let err = client.try_upgrade(&admin, &wasm_hash).unwrap_err().unwrap(); assert_eq!(err, ContractError::NotInitialized); } @@ -1032,10 +1054,7 @@ mod tests { client.initialize(&admin); - let err = client - .try_upgrade(&other, &wasm_hash) - .unwrap_err() - .unwrap(); + let err = client.try_upgrade(&other, &wasm_hash).unwrap_err().unwrap(); assert_eq!(err, ContractError::Unauthorized); } @@ -1078,7 +1097,10 @@ mod tests { // Document still verifiable after migration. assert!(client.verify_document(&document_hash)); - assert_eq!(client.get_document_status(&document_hash), DocumentStatus::Active); + assert_eq!( + client.get_document_status(&document_hash), + DocumentStatus::Active + ); } // --- feature flags --- diff --git a/src/main.rs b/src/main.rs index f239d17..01804fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -92,18 +92,20 @@ mod native { /// `POST /config/reload` — triggers a config reload from environment variables. async fn config_reload_handler(State(state): State) -> impl IntoResponse { match AppConfig::from_env_with_metrics(Some(Arc::clone(&state.metrics))) { - Ok(new_config) => { - match ConfigUpdate::new(new_config) { - Ok(update) => { - if state.config_watcher.send(update).is_ok() { - Json(json!({"status": "ok", "message": "config reload triggered successfully"})) - } else { - Json(json!({"status": "error", "message": "no subscribers for config update"})) - } + Ok(new_config) => match ConfigUpdate::new(new_config) { + Ok(update) => { + if state.config_watcher.send(update).is_ok() { + Json( + json!({"status": "ok", "message": "config reload triggered successfully"}), + ) + } else { + Json( + json!({"status": "error", "message": "no subscribers for config update"}), + ) } - Err(e) => Json(json!({"status": "error", "message": e.to_string()})), } - } + Err(e) => Json(json!({"status": "error", "message": e.to_string()})), + }, Err(e) => Json(json!({"status": "error", "message": e.to_string()})), } } @@ -150,14 +152,12 @@ mod native { } )) } - CacheBackend::Redis(_) => { - Json(json!( - { - "backend": "redis", - "message": "Redis cache statistics not yet implemented" - } - )) - } + CacheBackend::Redis(_) => Json(json!( + { + "backend": "redis", + "message": "Redis cache statistics not yet implemented" + } + )), } } @@ -170,7 +170,10 @@ mod native { let config = AppConfig::from_env_with_metrics(Some(Arc::clone(&metrics))) .map_err(|e| anyhow::anyhow!("{e}"))?; - eprintln!("[proofstell] Configuration v{} loaded successfully", AppConfig::version()); + eprintln!( + "[proofstell] Configuration v{} loaded successfully", + AppConfig::version() + ); eprintln!("[proofstell] port: {}", config.port); eprintln!( "[proofstell] stellar_horizon_url: {}", @@ -188,8 +191,7 @@ mod native { ); eprintln!( "[proofstell] cache: backend={}, max_size={}", - config.cache_backend, - config.cache_max_size + config.cache_backend, config.cache_max_size ); // ── Create config hot-reload channel ───────────────────────── @@ -214,7 +216,10 @@ mod native { } } _ => { - eprintln!("[proofstell] Initializing InMemory cache backend (max_size={})", config.cache_max_size); + eprintln!( + "[proofstell] Initializing InMemory cache backend (max_size={})", + config.cache_max_size + ); let cache = InMemoryCache::with_max_size(config.cache_max_size) .with_metrics(Arc::clone(&metrics)); Arc::new(CacheBackend::InMemory(cache)) diff --git a/src/metrics.rs b/src/metrics.rs index 95a8d58..8fb0ea2 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1,6 +1,6 @@ use prometheus::{ - Counter, Encoder, Gauge, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, - Opts, Registry, TextEncoder, + Counter, Encoder, Gauge, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, Opts, + Registry, TextEncoder, }; use std::prelude::v1::*; use std::sync::Arc; @@ -56,6 +56,11 @@ pub struct MetricsRegistry { config_validation_failures: IntCounter, config_reload_total: IntCounter, + // ── Circuit breaker metrics ── + circuit_state: Gauge, + circuit_transitions_total: IntCounterVec, + circuit_state_changes_total: IntCounterVec, + // ── Webhook delivery metrics ── webhook_deliveries_total: IntCounterVec, webhook_delivery_latency_seconds: HistogramVec, @@ -74,8 +79,7 @@ impl MetricsRegistry { let registry = Registry::new(); // ── General request metrics ── - let request_count = - Counter::new("requests_total", "Total number of API requests").unwrap(); + let request_count = Counter::new("requests_total", "Total number of API requests").unwrap(); let error_count = Counter::new("errors_total", "Total number of errors encountered").unwrap(); @@ -93,7 +97,8 @@ impl MetricsRegistry { ) .unwrap(); let cache_size = Gauge::new("cache_size", "Current number of entries in cache").unwrap(); - let cache_evictions = IntCounter::new("cache_evictions_total", "Total cache evictions").unwrap(); + let cache_evictions = + IntCounter::new("cache_evictions_total", "Total cache evictions").unwrap(); let cache_hit_rate = Gauge::new("cache_hit_rate", "Current cache hit rate (0-1)").unwrap(); // ── Document metrics ── @@ -140,9 +145,11 @@ impl MetricsRegistry { ) .unwrap(); - let retry_total = - IntCounter::new("retry_total", "Total number of retry attempts across all operations") - .unwrap(); + let retry_total = IntCounter::new( + "retry_total", + "Total number of retry attempts across all operations", + ) + .unwrap(); // ── Rate limiter metrics (legacy) ── let rate_limit_tokens_consumed = IntCounter::new( @@ -211,6 +218,31 @@ impl MetricsRegistry { ) .unwrap(); + // ── Circuit breaker metrics ───────────────────────────────────── + let circuit_state = Gauge::new( + "circuit_breaker_state", + "Current circuit breaker state (0=closed, 1=open, 2=half_open)", + ) + .unwrap(); + + let circuit_transitions_total = IntCounterVec::new( + Opts::new( + "circuit_breaker_transitions_total", + "Total circuit breaker state transitions by target state", + ), + &["to_state"], + ) + .unwrap(); + + let circuit_state_changes_total = IntCounterVec::new( + Opts::new( + "circuit_breaker_state_changes_total", + "Total circuit breaker state changes by from_state and to_state", + ), + &["from_state", "to_state"], + ) + .unwrap(); + // ── Webhook delivery metrics ── let webhook_deliveries_total = IntCounterVec::new( Opts::new( @@ -268,6 +300,9 @@ impl MetricsRegistry { Box::new(event_backlog_size.clone()), Box::new(config_validation_failures.clone()), Box::new(config_reload_total.clone()), + Box::new(circuit_state.clone()), + Box::new(circuit_transitions_total.clone()), + Box::new(circuit_state_changes_total.clone()), Box::new(webhook_deliveries_total.clone()), Box::new(webhook_delivery_latency_seconds.clone()), Box::new(webhook_dlq_depth.clone()), @@ -302,6 +337,9 @@ impl MetricsRegistry { event_backlog_size, config_validation_failures, config_reload_total, + circuit_state, + circuit_transitions_total, + circuit_state_changes_total, webhook_deliveries_total, webhook_delivery_latency_seconds, webhook_dlq_depth, @@ -405,9 +443,7 @@ impl MetricsRegistry { /// Record an accepted request for `issuer`. pub fn increment_rate_limit_hit(&self, issuer: &str) { - self.rate_limit_hits - .with_label_values(&[issuer]) - .inc(); + self.rate_limit_hits.with_label_values(&[issuer]).inc(); } /// Record a rejection originating from the **global** tier. @@ -456,7 +492,25 @@ impl MetricsRegistry { self.config_reload_total.inc(); } - // ── Webhook delivery metrics ───────────────────────────────────────── + // ── Circuit breaker metrics ────────────────────────────────────── + + pub fn set_circuit_state(&self, state: i64) { + self.circuit_state.set(state as f64); + } + + pub fn record_circuit_transition(&self, to_state: &str) { + self.circuit_transitions_total + .with_label_values(&[to_state]) + .inc(); + } + + pub fn record_circuit_state_change(&self, from_state: &str, to_state: &str) { + self.circuit_state_changes_total + .with_label_values(&[from_state, to_state]) + .inc(); + } + + // ── Webhook delivery metrics ────────────────────────────────────── /// Record a completed delivery attempt (success or dead_lettered) with latency. pub fn record_webhook_delivery(&self, status: &str, latency_secs: f64) { @@ -596,4 +650,4 @@ mod tests { let output = metrics.render(); assert!(output.contains("requests_total")); } -} \ No newline at end of file +} diff --git a/src/rate_limit.rs b/src/rate_limit.rs index e447546..790a42f 100644 --- a/src/rate_limit.rs +++ b/src/rate_limit.rs @@ -4,7 +4,12 @@ use governor::{ state::keyed::DefaultKeyedStateStore, Quota, RateLimiter, }; -use std::{num::NonZeroU32, string::{String, ToString}, sync::Arc, time::Duration}; +use std::{ + num::NonZeroU32, + string::{String, ToString}, + sync::Arc, + time::Duration, +}; use crate::{cache::CacheKey, metrics::MetricsRegistry}; @@ -18,18 +23,11 @@ fn now_secs() -> u64 { // ── Type aliases ───────────────────────────────────────────────────────────── /// Global (unkeyed) rate limiter backed by the Quanta monotonic clock. -pub type GlobalRateLimiter = RateLimiter< - governor::state::NotKeyed, - governor::state::InMemoryState, - QuantaClock, ->; +pub type GlobalRateLimiter = + RateLimiter; /// Per-issuer (keyed) rate limiter backed by the Quanta monotonic clock. -pub type KeyedRateLimiterInner = RateLimiter< - String, - DefaultKeyedStateStore, - QuantaClock, ->; +pub type KeyedRateLimiterInner = RateLimiter, QuantaClock>; // ── Rate limit status ───────────────────────────────────────────────────────── @@ -243,7 +241,8 @@ impl PerIssuerRateLimiter { /// Call this periodically (e.g. from a background task) to bound memory use. pub fn evict_stale(&self) { let cutoff = now_secs().saturating_sub(self.config.issuer_ttl_seconds); - self.issuer_meta.retain(|_, entry| entry.last_seen >= cutoff); + self.issuer_meta + .retain(|_, entry| entry.last_seen >= cutoff); } /// Return the number of tracked issuers currently in the metadata map. @@ -330,7 +329,10 @@ impl std::fmt::Display for RateLimitError { "global rate limit exceeded; retry after {}s", retry_after.as_secs() ), - Self::IssuerExhausted { issuer, retry_after } => write!( + Self::IssuerExhausted { + issuer, + retry_after, + } => write!( f, "per-issuer rate limit exceeded for '{}'; retry after {}s", issuer, @@ -513,4 +515,4 @@ mod tests { // 4th should be rejected assert!(limiter.check("burst-issuer").is_err()); } -} \ No newline at end of file +} diff --git a/src/stellar.rs b/src/stellar.rs index 8e51f4b..2832c3e 100644 --- a/src/stellar.rs +++ b/src/stellar.rs @@ -1,18 +1,25 @@ use anyhow::Result; +use rand::Rng; use serde::{Deserialize, Serialize}; use std::{ boxed::Box, fmt, future::Future, string::{String, ToString}, - sync::{Arc, Mutex}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, time::{Duration, Instant}, vec::Vec, }; use thiserror::Error; +use tokio::sync::Semaphore as AsyncSemaphore; use crate::{ - cache::{CacheBackend, CacheKey}, config::AppConfig, hash_validator::CanonicalHash, + cache::{CacheBackend, CacheKey}, + config::AppConfig, + hash_validator::CanonicalHash, }; use crate::metrics::MetricsRegistry; @@ -33,6 +40,7 @@ pub struct StellarClient { metrics: Option>, config: StellarClientConfig, cache: Option>, + bulkhead: Arc, } #[derive(Debug, Clone)] @@ -42,6 +50,8 @@ pub struct StellarClientConfig { pub request_timeout: Duration, pub rate_limit_per_second: u32, pub rate_limit_burst: u32, + pub bulkhead: BulkheadConfig, + pub graceful_degradation: GracefulDegradationConfig, } #[derive(Debug, Clone)] @@ -56,6 +66,8 @@ pub struct RetryPolicy { pub enum RetryJitter { None, Full, + Equal, + Decorrelated, } #[derive(Debug, Clone)] @@ -65,6 +77,36 @@ pub struct CircuitBreakerConfig { pub half_open_max_calls: u32, } +#[derive(Debug, Clone)] +pub struct BulkheadConfig { + pub max_concurrent: u32, + pub max_queue: u32, +} + +#[derive(Debug, Clone)] +pub struct GracefulDegradationConfig { + pub fallback_cache_ttl: Duration, + pub stale_cache_ok: bool, +} + +impl Default for BulkheadConfig { + fn default() -> Self { + Self { + max_concurrent: 10, + max_queue: 100, + } + } +} + +impl Default for GracefulDegradationConfig { + fn default() -> Self { + Self { + fallback_cache_ttl: Duration::from_secs(300), + stale_cache_ok: true, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CircuitState { Closed, @@ -81,6 +123,10 @@ pub struct CircuitBreakerMetrics { pub rejected_calls: u64, pub successful_calls: u64, pub failed_calls: u64, + pub timeout_calls: u64, + pub retryable_http_calls: u64, + pub consecutive_failures: u32, + pub in_flight: u32, } #[derive(Debug, Error)] @@ -208,6 +254,7 @@ impl StellarClient { circuit_breaker: Arc::new(CircuitBreaker::new(config.circuit_breaker.clone())), max_retries: config.retry.max_retries, metrics: None, + bulkhead: Arc::new(AsyncSemaphore::new(config.bulkhead.max_concurrent as usize)), config, cache: None, } @@ -235,7 +282,10 @@ impl StellarClient { /// Warm the cache with pre-known verification results. /// This is useful for loading frequently accessed hashes at startup. - pub async fn warm_cache(&self, entries: Vec<(String, VerificationResult, u64)>) -> Result { + pub async fn warm_cache( + &self, + entries: Vec<(String, VerificationResult, u64)>, + ) -> Result { if let Some(cache) = &self.cache { if let CacheBackend::InMemory(inmem) = cache.as_ref() { let cache_entries: Vec<_> = entries @@ -246,7 +296,7 @@ impl StellarClient { (key, value, ttl) }) .collect(); - + inmem.warm(cache_entries).await } else { Ok(0) // Redis warming not implemented yet @@ -326,7 +376,10 @@ impl StellarClient { .await } - async fn execute_verify_hash_with_retry(&self, hash: &str) -> StellarResult { + async fn execute_verify_hash_with_retry( + &self, + hash: &str, + ) -> StellarResult { let result = self.verify_hash(hash).await; match result.status { VerificationStatus::ConfirmedMatch | VerificationStatus::NoMatch => Ok(result), @@ -349,18 +402,17 @@ impl StellarClient { } fn retry_delay(&self, attempt: u32) -> Duration { + let base = self.config.retry.base_delay; + let max_delay = self.config.retry.max_delay; let multiplier = 2_u32.saturating_pow(attempt.min(31)); - let exponential = self - .config - .retry - .base_delay - .checked_mul(multiplier) - .unwrap_or(self.config.retry.max_delay); - let capped = exponential.min(self.config.retry.max_delay); + let exponential = base.checked_mul(multiplier).unwrap_or(max_delay); + let capped = exponential.min(max_delay); match self.config.retry.jitter { RetryJitter::None => capped, - RetryJitter::Full => jittered_delay(capped), + RetryJitter::Full => jittered_delay_full(capped), + RetryJitter::Equal => jittered_delay_equal(capped), + RetryJitter::Decorrelated => jittered_delay_decorrelated(capped, attempt), } } @@ -403,11 +455,25 @@ impl StellarClient { if let Some(ref m) = self.metrics { m.increment_retry(); } - tokio::time::sleep(Duration::from_millis(200 * attempt as u64)).await; + tokio::time::sleep(self.retry_delay(attempt - 1)).await; } let horizon_start = MetricsRegistry::start_timer(); let url = format!("{}/transactions?memo={}", self.horizon_url, hash); + + let bulkhead_acquired = + tokio::time::timeout(Duration::from_millis(500), self.bulkhead.acquire()) + .await + .is_ok(); + + if !bulkhead_acquired { + if let Some(fallback) = self.try_stale_cache_fallback(hash).await { + return fallback; + } + last_status = VerificationStatus::NetworkError; + continue; + } + let resp_result = self.http_client.get(&url).send().await; match resp_result { @@ -467,7 +533,7 @@ impl StellarClient { m.record_horizon_latency("error", horizon_latency); } last_status = VerificationStatus::NetworkError; - // Network error — continue retry + last_http_status = None; } } } @@ -558,6 +624,16 @@ impl StellarClient { Ok(None) } + async fn try_stale_cache_fallback(&self, hash: &str) -> Option { + let cache = self.cache.as_ref()?; + let key = CacheKey::verification(hash); + let result = cache.get::(&key).await.ok()??; + if result.verified() || self.config.graceful_degradation.stale_cache_ok { + return Some(result); + } + None + } + pub async fn anchor_transfer(&self, _transfer_hash: &str, _memo: &str) -> Result<()> { if let Some(ref m) = self.metrics { m.increment_request_count(); @@ -573,20 +649,29 @@ impl StellarClientConfig { max_retries: config.stellar_max_retries, base_delay: Duration::from_millis(config.stellar_retry_base_delay_ms), max_delay: Duration::from_millis(config.stellar_retry_max_delay_ms), - jitter: if config.stellar_retry_jitter_enabled { - RetryJitter::Full - } else { - RetryJitter::None + jitter: match config.stellar_retry_jitter_type.to_lowercase().as_str() { + "none" => RetryJitter::None, + "full" => RetryJitter::Full, + "equal" => RetryJitter::Equal, + "decorrelated" => RetryJitter::Decorrelated, + _ => RetryJitter::Full, }, }, circuit_breaker: CircuitBreakerConfig { failure_threshold: config.stellar_circuit_breaker_failure_threshold, - open_duration: Duration::from_millis(config.stellar_circuit_breaker_open_duration_ms), + open_duration: Duration::from_millis( + config.stellar_circuit_breaker_open_duration_ms, + ), half_open_max_calls: config.stellar_circuit_breaker_half_open_max_calls, }, request_timeout: Duration::from_millis(config.stellar_request_timeout_ms), rate_limit_per_second: config.rate_limit_per_second, rate_limit_burst: config.rate_limit_burst, + bulkhead: BulkheadConfig { + max_concurrent: config.stellar_bulkhead_max_concurrent, + max_queue: config.stellar_bulkhead_max_queue, + }, + graceful_degradation: GracefulDegradationConfig::default(), } } } @@ -608,6 +693,8 @@ impl Default for StellarClientConfig { request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MS), rate_limit_per_second: 10, rate_limit_burst: 10, + bulkhead: BulkheadConfig::default(), + graceful_degradation: GracefulDegradationConfig::default(), } } } @@ -631,8 +718,7 @@ impl StellarError { | Self::RetryableHttpStatus { .. } | Self::ResponseParse { .. } => true, Self::RetryExhausted { final_error, .. } => final_error.is_retryable(), - Self::NonRetryableHttpStatus { .. } - | Self::VerificationNotFound { .. } => false, + Self::NonRetryableHttpStatus { .. } | Self::VerificationNotFound { .. } => false, } } @@ -648,29 +734,37 @@ impl StellarError { #[derive(Debug)] struct CircuitBreaker { config: CircuitBreakerConfig, - state: Mutex, + state: AtomicUsize, opened_at: Mutex>, - consecutive_failures: Mutex, - half_open_in_flight: Mutex, - half_open_successes: Mutex, + consecutive_failures: AtomicUsize, + half_open_in_flight: AtomicUsize, + half_open_successes: AtomicUsize, + half_open_semaphore: Arc, metrics: Mutex, } impl CircuitBreaker { fn new(config: CircuitBreakerConfig) -> Self { + let max_calls = config.half_open_max_calls.max(1) as usize; Self { config, - state: Mutex::new(CircuitState::Closed), + state: AtomicUsize::new(CircuitState::Closed as usize), opened_at: Mutex::new(None), - consecutive_failures: Mutex::new(0), - half_open_in_flight: Mutex::new(0), - half_open_successes: Mutex::new(0), + consecutive_failures: AtomicUsize::new(0), + half_open_in_flight: AtomicUsize::new(0), + half_open_successes: AtomicUsize::new(0), + half_open_semaphore: Arc::new(AsyncSemaphore::new(max_calls)), metrics: Mutex::new(CircuitBreakerMetrics::default()), } } fn state(&self) -> CircuitState { - *self.state.lock().unwrap() + match self.state.load(Ordering::Relaxed) { + 0 => CircuitState::Closed, + 1 => CircuitState::Open, + 2 => CircuitState::HalfOpen, + _ => CircuitState::Closed, + } } fn metrics(&self) -> CircuitBreakerMetrics { @@ -682,131 +776,152 @@ impl CircuitBreaker { F: FnOnce() -> Fut, Fut: Future>, { - self.allow_call()?; + self.allow_call().await?; let result = operation().await; - self.record_result(&result); + self.record_result(&result).await; result } - fn allow_call(&self) -> StellarResult<()> { - let mut state = self.state.lock().unwrap(); - - match *state { - CircuitState::Closed => Ok(()), - CircuitState::Open => { - let opened_at = *self.opened_at.lock().unwrap(); - if let Some(opened_at) = opened_at { - let elapsed = opened_at.elapsed(); - if elapsed >= self.config.open_duration { - *state = CircuitState::HalfOpen; - *self.half_open_in_flight.lock().unwrap() = 0; - *self.half_open_successes.lock().unwrap() = 0; + async fn allow_call(&self) -> StellarResult<()> { + loop { + let current = self.state.load(Ordering::Relaxed); + match current { + s if s == (CircuitState::Closed as usize) => return Ok(()), + s if s == (CircuitState::Open as usize) => { + let opened_at = *self.opened_at.lock().unwrap(); + if let Some(opened_at) = opened_at { + let elapsed = opened_at.elapsed(); + if elapsed >= self.config.open_duration { + let expected = CircuitState::HalfOpen as usize; + if self + .state + .compare_exchange( + current, + expected, + Ordering::Relaxed, + Ordering::Relaxed, + ) + .is_ok() + { + self.half_open_in_flight.store(0, Ordering::Relaxed); + self.half_open_successes.store(0, Ordering::Relaxed); + return Ok(()); + } + continue; + } + let retry_after = self.config.open_duration.saturating_sub(elapsed); + self.increment_rejected_calls(); + return Err(StellarError::CircuitOpen { + state: CircuitState::Open, + retry_after, + }); + } + let expected = CircuitState::HalfOpen as usize; + if self + .state + .compare_exchange(current, expected, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + self.half_open_in_flight.store(0, Ordering::Relaxed); + self.half_open_successes.store(0, Ordering::Relaxed); return Ok(()); } - - let retry_after = self.config.open_duration.saturating_sub(elapsed); - self.increment_rejected_calls(); - return Err(StellarError::CircuitOpen { - state: *state, - retry_after, - }); + continue; } - - *state = CircuitState::HalfOpen; - Ok(()) - } - CircuitState::HalfOpen => { - let max_calls = self.config.half_open_max_calls.max(1); - let mut in_flight = self.half_open_in_flight.lock().unwrap(); - if *in_flight >= max_calls { - self.increment_rejected_calls(); - Err(StellarError::CircuitOpen { - state: *state, - retry_after: Duration::ZERO, - }) - } else { - *in_flight += 1; - Ok(()) + s if s == (CircuitState::HalfOpen as usize) => { + let _max_calls = self.config.half_open_max_calls.max(1) as usize; + let permit = self.half_open_semaphore.try_acquire(); + if permit.is_err() { + self.increment_rejected_calls(); + return Err(StellarError::CircuitOpen { + state: CircuitState::HalfOpen, + retry_after: Duration::ZERO, + }); + } + self.half_open_in_flight.fetch_add(1, Ordering::Relaxed); + return Ok(()); } + _ => return Ok(()), } } } - fn record_result(&self, result: &StellarResult) { + async fn record_result(&self, result: &StellarResult) { match result { - Ok(_) => self.record_success(), - Err(err) if err.affects_circuit_breaker() => self.record_failure(), + Ok(_) => self.record_success().await, + Err(err) if err.affects_circuit_breaker() => self.record_failure(err).await, Err(_) => {} } } - fn record_success(&self) { - let state = *self.state.lock().unwrap(); - - match state { - CircuitState::Closed => { - *self.consecutive_failures.lock().unwrap() = 0; - self.increment_successful_calls(); - } - CircuitState::HalfOpen => { - { - let mut in_flight = self.half_open_in_flight.lock().unwrap(); - *in_flight = in_flight.saturating_sub(1); - } - - let should_close; - { - let mut successes = self.half_open_successes.lock().unwrap(); - *successes = successes.saturating_add(1); - should_close = *successes >= self.config.half_open_max_calls.max(1); - } + async fn record_success(&self) { + let current = self.state.load(Ordering::Relaxed); + if current == (CircuitState::Closed as usize) { + self.consecutive_failures.store(0, Ordering::Relaxed); + self.increment_successful_calls(); + return; + } + if current == (CircuitState::HalfOpen as usize) { + self.half_open_in_flight.fetch_sub(1, Ordering::Relaxed); + let successes = self.half_open_successes.fetch_add(1, Ordering::Relaxed) + 1; + let should_close = + successes >= self.config.half_open_max_calls.max(1).try_into().unwrap(); + self.increment_successful_calls(); + self.increment_half_open_successes(); + if should_close { + let expected = CircuitState::HalfOpen as usize; + let closed = CircuitState::Closed as usize; + if self + .state + .compare_exchange(expected, closed, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() { - let mut metrics = self.metrics.lock().unwrap(); - metrics.half_open_successes = metrics.half_open_successes.saturating_add(1); - } - self.increment_successful_calls(); - - if should_close { - *self.state.lock().unwrap() = CircuitState::Closed; - *self.opened_at.lock().unwrap() = None; - *self.consecutive_failures.lock().unwrap() = 0; + self.opened_at.lock().unwrap().take(); + self.consecutive_failures.store(0, Ordering::Relaxed); self.increment_recoveries(); } } - CircuitState::Open => {} } } - fn record_failure(&self) { - let state = *self.state.lock().unwrap(); - - match state { - CircuitState::Closed => { - let mut failures = self.consecutive_failures.lock().unwrap(); - *failures = failures.saturating_add(1); - self.increment_failed_calls(); - - if *failures >= self.config.failure_threshold.max(1) { - *self.state.lock().unwrap() = CircuitState::Open; + async fn record_failure(&self, err: &StellarError) { + let current = self.state.load(Ordering::Relaxed); + if current == (CircuitState::Closed as usize) { + let failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed) + 1; + match err { + StellarError::Timeout { .. } => self.increment_timeout_calls(), + StellarError::RetryableHttpStatus { .. } => self.increment_retryable_http_calls(), + _ => {} + } + self.increment_failed_calls(); + if failures >= self.config.failure_threshold.max(1).try_into().unwrap() { + let expected = CircuitState::Closed as usize; + let open = CircuitState::Open as usize; + if self + .state + .compare_exchange(expected, open, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { *self.opened_at.lock().unwrap() = Some(Instant::now()); self.increment_trips(); } } - CircuitState::HalfOpen => { - { - let mut in_flight = self.half_open_in_flight.lock().unwrap(); - *in_flight = in_flight.saturating_sub(1); - } - { - let mut metrics = self.metrics.lock().unwrap(); - metrics.half_open_failures = metrics.half_open_failures.saturating_add(1); - } - self.increment_failed_calls(); - *self.state.lock().unwrap() = CircuitState::Open; + return; + } + if current == (CircuitState::HalfOpen as usize) { + self.half_open_in_flight.fetch_sub(1, Ordering::Relaxed); + let expected = CircuitState::HalfOpen as usize; + let open = CircuitState::Open as usize; + if self + .state + .compare_exchange(expected, open, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { *self.opened_at.lock().unwrap() = Some(Instant::now()); self.increment_trips(); + self.increment_failed_calls(); + self.increment_half_open_failures(); } - CircuitState::Open => {} } } @@ -834,6 +949,26 @@ impl CircuitBreaker { let mut metrics = self.metrics.lock().unwrap(); metrics.failed_calls = metrics.failed_calls.saturating_add(1); } + + fn increment_timeout_calls(&self) { + let mut metrics = self.metrics.lock().unwrap(); + metrics.timeout_calls = metrics.timeout_calls.saturating_add(1); + } + + fn increment_retryable_http_calls(&self) { + let mut metrics = self.metrics.lock().unwrap(); + metrics.retryable_http_calls = metrics.retryable_http_calls.saturating_add(1); + } + + fn increment_half_open_successes(&self) { + let mut metrics = self.metrics.lock().unwrap(); + metrics.half_open_successes = metrics.half_open_successes.saturating_add(1); + } + + fn increment_half_open_failures(&self) { + let mut metrics = self.metrics.lock().unwrap(); + metrics.half_open_failures = metrics.half_open_failures.saturating_add(1); + } } impl fmt::Display for CircuitState { @@ -854,7 +989,7 @@ fn is_retryable_status(status: u16) -> bool { status == 408 || status == 429 || (500..=599).contains(&status) } -fn jittered_delay(max_delay: Duration) -> Duration { +fn jittered_delay_full(max_delay: Duration) -> Duration { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() @@ -864,6 +999,19 @@ fn jittered_delay(max_delay: Duration) -> Duration { Duration::from_millis(millis) } +fn jittered_delay_equal(max_delay: Duration) -> Duration { + let millis = (max_delay.as_secs_f64() * 1000.0 / 2.0).round() as u64; + Duration::from_millis(millis) +} + +fn jittered_delay_decorrelated(base: Duration, _attempt: u32) -> Duration { + let mut rng = rand::thread_rng(); + let millis = base.as_millis() as u64; + let decorrelated = millis.saturating_mul(3).saturating_div(4); + let jitter = rng.gen_range(0..decorrelated.max(1)); + Duration::from_millis(jitter.min(base.as_millis() as u64)) +} + async fn sleep(delay: Duration) { if delay.is_zero() { tokio::task::yield_now().await; @@ -895,6 +1043,8 @@ mod tests { request_timeout: Duration::from_secs(2), rate_limit_per_second: 100, rate_limit_burst: 100, + bulkhead: BulkheadConfig::default(), + graceful_degradation: GracefulDegradationConfig::default(), } } @@ -1009,10 +1159,7 @@ mod tests { let err = client.verify_hash_with_retry(hash).await.unwrap_err(); match err { - StellarError::CircuitOpen { - state, - retry_after, - } => { + StellarError::CircuitOpen { state, retry_after } => { assert_eq!(state, CircuitState::Open); assert!(retry_after <= Duration::from_millis(50)); } @@ -1157,18 +1304,15 @@ mod tests { Mock::given(method("GET")) .and(path("transactions")) .and(query_param("memo", hash)) - .respond_with(ResponseTemplate::new(200).set_body_json( - horizon_tx_json( - "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", - hash, - "2024-01-15T10:30:00Z", - ), - )) + .respond_with(ResponseTemplate::new(200).set_body_json(horizon_tx_json( + "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + hash, + "2024-01-15T10:30:00Z", + ))) .mount(&server) .await; - let client = StellarClient::new(&server.uri()) - .with_max_retries(0); + let client = StellarClient::new(&server.uri()).with_max_retries(0); let result = client.verify_hash(hash).await; @@ -1189,8 +1333,7 @@ mod tests { .mount(&server) .await; - let client = StellarClient::new(&server.uri()) - .with_max_retries(0); + let client = StellarClient::new(&server.uri()).with_max_retries(0); let result = client.verify_hash(hash).await; @@ -1207,18 +1350,15 @@ mod tests { Mock::given(method("GET")) .and(path("transactions")) .and(query_param("memo", hash)) - .respond_with(ResponseTemplate::new(200).set_body_json( - horizon_tx_json( - "tx123", - "wrong-hash-0000000000000000000000000000000000000000000000000000", - "2024-01-15T10:30:00Z", - ), - )) + .respond_with(ResponseTemplate::new(200).set_body_json(horizon_tx_json( + "tx123", + "wrong-hash-0000000000000000000000000000000000000000000000000000", + "2024-01-15T10:30:00Z", + ))) .mount(&server) .await; - let client = StellarClient::new(&server.uri()) - .with_max_retries(0); + let client = StellarClient::new(&server.uri()).with_max_retries(0); let result = client.verify_hash(hash).await; @@ -1237,8 +1377,7 @@ mod tests { .mount(&server) .await; - let client = StellarClient::new(&server.uri()) - .with_max_retries(0); + let client = StellarClient::new(&server.uri()).with_max_retries(0); let result = client.verify_hash(hash).await; @@ -1253,14 +1392,11 @@ mod tests { Mock::given(method("GET")) .and(path("transactions")) - .respond_with( - ResponseTemplate::new(200).set_body_string("not-valid-json{{{") - ) + .respond_with(ResponseTemplate::new(200).set_body_string("not-valid-json{{{")) .mount(&server) .await; - let client = StellarClient::new(&server.uri()) - .with_max_retries(0); + let client = StellarClient::new(&server.uri()).with_max_retries(0); let result = client.verify_hash(hash).await; @@ -1286,9 +1422,11 @@ mod tests { Mock::given(method("GET")) .and(path("transactions")) .and(query_param("memo", hash)) - .respond_with(ResponseTemplate::new(200).set_body_json( - horizon_tx_json("tx-retry-ok", hash, "2024-01-15T10:30:00Z"), - )) + .respond_with(ResponseTemplate::new(200).set_body_json(horizon_tx_json( + "tx-retry-ok", + hash, + "2024-01-15T10:30:00Z", + ))) .mount(&server) .await; @@ -1314,8 +1452,7 @@ mod tests { .mount(&server) .await; - let client = StellarClient::new(&server.uri()) - .with_max_retries(3); + let client = StellarClient::new(&server.uri()).with_max_retries(3); let result = client.verify_hash(hash).await; diff --git a/src/webhook.rs b/src/webhook.rs index 4df4508..6d4323a 100644 --- a/src/webhook.rs +++ b/src/webhook.rs @@ -30,8 +30,8 @@ type HmacSha256 = Hmac; /// The signature is computed over the serialized JSON body using the configured /// webhook secret. Receivers can verify the signature using the shared secret. fn compute_webhook_signature(secret: &str, body: &[u8]) -> String { - let mut mac = HmacSha256::new_from_slice(secret.as_bytes()) - .expect("HMAC-SHA256 accepts any key length"); + let mut mac = + HmacSha256::new_from_slice(secret.as_bytes()).expect("HMAC-SHA256 accepts any key length"); mac.update(body); let result = mac.finalize(); let code = result.into_bytes(); @@ -242,7 +242,7 @@ impl WebhookDispatcher { if let Some(cache) = &self.cache { let dedup_key = format!("{}{}", DEDUP_KEY_PREFIX, event.idempotency_key); let cache_key = CacheKey::Events(dedup_key); - + if let Ok(Some(_)) = cache.get_raw(&cache_key).await { // Already delivered, skip if let Some(ref m) = self.metrics { @@ -250,9 +250,11 @@ impl WebhookDispatcher { } return; } - + // Mark as delivered with TTL - let _ = cache.set_raw(&cache_key, "delivered", self.deduplication_ttl).await; + let _ = cache + .set_raw(&cache_key, "delivered", self.deduplication_ttl) + .await; } let payload = WebhookPayload::from(event); @@ -456,7 +458,7 @@ fn jitter_ms(max_ms: u64) -> u64 { #[cfg(test)] mod tests { use super::*; - use crate::cache::{CacheBackend, InMemoryCache}; + use crate::cache::CacheBackend; use std::sync::Arc; use wiremock::matchers::{header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -952,13 +954,15 @@ mod tests { ..Default::default() }; - let dispatcher = WebhookDispatcher::new(config, None) - .with_cache(Arc::clone(&cache)); + let dispatcher = WebhookDispatcher::new(config, None).with_cache(Arc::clone(&cache)); dispatcher.dispatch(&make_event()).await; let dlq_key = CacheKey::Events(format!("{}{}", DLQ_REDIS_KEY_PREFIX, server.uri())); let persisted = cache.get_raw(&dlq_key).await.unwrap(); - assert!(persisted.is_some(), "DLQ entry should be persisted to cache"); + assert!( + persisted.is_some(), + "DLQ entry should be persisted to cache" + ); let entry: DeadLetterEntry = serde_json::from_str(&persisted.unwrap()).unwrap(); assert_eq!(entry.url, server.uri()); @@ -982,8 +986,7 @@ mod tests { ..Default::default() }; - let dispatcher = WebhookDispatcher::new(config, None) - .with_cache(Arc::clone(&cache)); + let dispatcher = WebhookDispatcher::new(config, None).with_cache(Arc::clone(&cache)); dispatcher.dispatch(&make_event()).await; // Drain returns the persisted entry (single source of truth when cache is available)