diff --git a/Cargo.lock b/Cargo.lock index 09cca410c..45af9f5d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 1.1.4", + "rustix", "slab", "windows-sys 0.61.2", ] @@ -365,25 +365,22 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.66.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags 2.11.1", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools 0.10.5", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash 2.1.2", "shlex", "syn 2.0.117", - "which", ] [[package]] @@ -643,9 +640,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.8.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" dependencies = [ "glob", "libc", @@ -973,7 +970,7 @@ dependencies = [ "crossterm_winapi", "document-features", "parking_lot", - "rustix 1.1.4", + "rustix", "winapi", ] @@ -1952,15 +1949,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "hound" version = "3.5.1" @@ -2498,12 +2486,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128fmt" version = "0.1.0" @@ -2551,12 +2533,6 @@ dependencies = [ "libc", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3669,12 +3645,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pegaflow-common" version = "0.23.5" @@ -3862,7 +3832,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -4356,9 +4326,9 @@ dependencies = [ [[package]] name = "rdma-mummy-sys" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8ead4d7835190d761b9365d9e53906a226d7699007b35bdca49d7edb89a15" +checksum = "7366fa21da4b5cb28f6301888f126770e87cbf8ec6400fe4b7c5d7ee7c5faa5c" dependencies = [ "bindgen", "cmake", @@ -4634,19 +4604,6 @@ dependencies = [ "transpose", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.11.1", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -4656,7 +4613,7 @@ dependencies = [ "bitflags 2.11.1", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -5165,9 +5122,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -5461,7 +5418,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -6742,18 +6699,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "win-sys" version = "0.3.1" @@ -6897,15 +6842,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" diff --git a/openinfer-engine/src/engine.rs b/openinfer-engine/src/engine.rs index 9086e714f..6be9da246 100644 --- a/openinfer-engine/src/engine.rs +++ b/openinfer-engine/src/engine.rs @@ -367,6 +367,18 @@ pub struct LoadSnapshot { pub num_running_reqs: u64, /// Requests admitted but not yet running (KV pressure, prefetch wait). pub num_waiting_reqs: u64, + /// Cumulative number of prompt tokens that queried the prefix cache. + /// + /// Feeds the upstream `SchedulerStats.prefix_cache_stats.base.queries` + /// counter (`prefix_cache_queries_total`). Left 0 by schedulers that do not + /// yet track it; the bridge maps it through regardless so the metric + /// pipeline is wired end-to-end. + pub prefix_cache_queries: u64, + /// Cumulative number of prompt tokens served from the prefix cache + /// (i.e. prefix-cache hits). Maps to + /// `SchedulerStats.prefix_cache_stats.base.hits` + /// (`prefix_cache_hits_total`). + pub prefix_cache_hits: u64, } /// One full KV block that just became reusable from this engine's prefix cache. diff --git a/openinfer-glm52/src/scheduler/load.rs b/openinfer-glm52/src/scheduler/load.rs index 19373a00e..862823e90 100644 --- a/openinfer-glm52/src/scheduler/load.rs +++ b/openinfer-glm52/src/scheduler/load.rs @@ -26,5 +26,6 @@ pub(super) fn publish_load( kv_total_blocks: kv_total_blocks as u64, num_running_reqs: slots.iter().flatten().count() as u64, num_waiting_reqs: pending.len() as u64, + ..Default::default() }); } diff --git a/openinfer-qwen3/src/scheduler.rs b/openinfer-qwen3/src/scheduler.rs index 2b09026c4..1a74b4c21 100644 --- a/openinfer-qwen3/src/scheduler.rs +++ b/openinfer-qwen3/src/scheduler.rs @@ -515,12 +515,17 @@ fn publish_load( executor: &E, num_running_reqs: u64, num_waiting_reqs: u64, + prefix_cache_queries: u64, + prefix_cache_hits: u64, ) { load_tx.send_replace(LoadSnapshot { kv_used_blocks: kv_total.saturating_sub(executor.available_blocks() as u64), kv_total_blocks: kv_total, num_running_reqs, num_waiting_reqs, + prefix_cache_queries, + prefix_cache_hits, + ..Default::default() }); } @@ -552,6 +557,10 @@ fn scheduler_loop( // Decode-overlap async prefill: pending requests whose prefill is in-flight // on the prefill overlap stream. `None` when no async prefill is running. let mut inflight_prefill_pending: Option> = None; + // Prefix-cache counters surfaced to the vLLM frontend metrics. Accumulated + // across the scheduler's lifetime from each step's first-chunk queries/hits. + let mut prefix_cache_queries: u64 = 0; + let mut prefix_cache_hits: u64 = 0; info!("Scheduler ready"); @@ -564,6 +573,8 @@ fn scheduler_loop( + prefilling.len() + inflight_prefill_pending.as_ref().map_or(0, Vec::len)) as u64, (deferred.len() + loading.len()) as u64, + prefix_cache_queries, + prefix_cache_hits, ); // Flush the prior step's cache changes to a router (no-op unless the // event feed is on). Top-of-loop, like `publish_load`: one pass per @@ -590,6 +601,8 @@ fn scheduler_loop( scheduled_at_unix_s, }; let effects = resolve_step(&executor, &active, artifacts); + prefix_cache_queries += effects.prefix_queries; + prefix_cache_hits += effects.prefix_hits; apply_effects( &mut executor, &mut active, @@ -724,6 +737,8 @@ fn scheduler_loop( // Only apply decode effects from the unified result. let effects = resolve_step(&executor, &active, artifacts); + prefix_cache_queries += effects.prefix_queries; + prefix_cache_hits += effects.prefix_hits; apply_effects( &mut executor, &mut active, @@ -754,6 +769,8 @@ fn scheduler_loop( } }; let effects = resolve_step(&executor, &active, artifacts); + prefix_cache_queries += effects.prefix_queries; + prefix_cache_hits += effects.prefix_hits; apply_effects( &mut executor, &mut active, @@ -783,6 +800,10 @@ fn scheduler_loop_with_lora_control( let mut pending_control: VecDeque = VecDeque::new(); let mut post_control_deferred: Vec = Vec::new(); let mut tracker = phase_trace::PhaseTracker::default(); + // Prefix-cache counters surfaced to the vLLM frontend metrics. Accumulated + // across the scheduler's lifetime from each step's first-chunk queries/hits. + let mut prefix_cache_queries: u64 = 0; + let mut prefix_cache_hits: u64 = 0; info!("Scheduler ready with LoRA control"); @@ -793,6 +814,8 @@ fn scheduler_loop_with_lora_control( &executor, (active.len() + prefilling.len()) as u64, (deferred.len() + loading.len() + post_control_deferred.len()) as u64, + prefix_cache_queries, + prefix_cache_hits, ); // 1. Drain incoming commands. Generation submitted after a pending @@ -925,6 +948,8 @@ fn scheduler_loop_with_lora_control( } }; let effects = resolve_step(&executor, &active, artifacts); + prefix_cache_queries += effects.prefix_queries; + prefix_cache_hits += effects.prefix_hits; apply_effects( &mut executor, &mut active, diff --git a/openinfer-qwen3/src/scheduler/effects.rs b/openinfer-qwen3/src/scheduler/effects.rs index 978a7ac9c..1c9b9db17 100644 --- a/openinfer-qwen3/src/scheduler/effects.rs +++ b/openinfer-qwen3/src/scheduler/effects.rs @@ -93,6 +93,15 @@ pub(super) struct StepEffects { pub(super) prompt_echoes: Vec, pub(super) pending: Vec, pub(super) decode: Vec, + /// Prefix-cache queries counted this step: one per request whose first + /// prefill chunk ran (a `queries` increment in vLLM terms). Carried into + /// `LoadSnapshot.prefix_cache_queries` for the vLLM frontend metrics. + pub(super) prefix_queries: u64, + /// Prefix-cache hit tokens counted this step: the sum of `cached_tokens` + /// across first-chunk requests (a `hits` increment in vLLM terms, token + /// granularity rather than block). Carried into + /// `LoadSnapshot.prefix_cache_hits` for the vLLM frontend metrics. + pub(super) prefix_hits: u64, } impl StepEffects { @@ -102,6 +111,8 @@ impl StepEffects { prompt_echoes: Vec::new(), pending: Vec::new(), decode: Vec::new(), + prefix_queries: 0, + prefix_hits: 0, } } } diff --git a/openinfer-qwen3/src/scheduler/resolve.rs b/openinfer-qwen3/src/scheduler/resolve.rs index 15418ecab..53919a6c8 100644 --- a/openinfer-qwen3/src/scheduler/resolve.rs +++ b/openinfer-qwen3/src/scheduler/resolve.rs @@ -29,12 +29,16 @@ pub(super) fn resolve_step( prompt_echoes: Vec::new(), pending: Vec::new(), decode: resolve_decode_outputs(executor, active, &result.requests), + prefix_queries: 0, + prefix_hits: 0, }, ExecutionArtifacts::SpeculativeDecode { verify } => StepEffects { scheduled: Vec::new(), prompt_echoes: Vec::new(), pending: Vec::new(), decode: resolve_speculative_outputs(executor, active, &verify.requests), + prefix_queries: 0, + prefix_hits: 0, }, ExecutionArtifacts::Unified { pending, @@ -126,6 +130,10 @@ fn resolve_prefill_outputs( prompt_tokens: prompt_len, cached_tokens: result.cached_tokens, }); + // First chunk is the only place a request counts toward the + // prefix-cache query total; its cached token span is the hit total. + effects.prefix_queries += 1; + effects.prefix_hits += result.cached_tokens as u64; } if !result.completed { diff --git a/openinfer-qwen3/src/scheduler/tests.rs b/openinfer-qwen3/src/scheduler/tests.rs index 01fa5993e..9db553c46 100644 --- a/openinfer-qwen3/src/scheduler/tests.rs +++ b/openinfer-qwen3/src/scheduler/tests.rs @@ -969,6 +969,8 @@ fn retiring_multiple_active_requests_tolerates_unsorted_indices() { scheduled: Vec::new(), prompt_echoes: Vec::new(), pending: Vec::new(), + prefix_queries: 0, + prefix_hits: 0, decode: vec![ effects::DecodeEffect::EmitAndFinish { request_id: RequestId(1), diff --git a/openinfer-qwen35/src/scheduler.rs b/openinfer-qwen35/src/scheduler.rs index c767eeee5..03c2c5694 100644 --- a/openinfer-qwen35/src/scheduler.rs +++ b/openinfer-qwen35/src/scheduler.rs @@ -841,6 +841,7 @@ fn publish_load( kv_total_blocks, num_running_reqs: (active.len() + prefilling.len()) as u64, num_waiting_reqs: num_waiting_reqs as u64, + ..Default::default() }); } diff --git a/openinfer-sim/tests/frontend_e2e.rs b/openinfer-sim/tests/frontend_e2e.rs index f1ebcd246..297851058 100644 --- a/openinfer-sim/tests/frontend_e2e.rs +++ b/openinfer-sim/tests/frontend_e2e.rs @@ -273,6 +273,9 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()> kv_total_blocks: 100, num_running_reqs: 1, num_waiting_reqs: 0, + prefix_cache_queries: 100, + prefix_cache_hits: 80, + ..Default::default() }, )?; server.publish_load( @@ -282,6 +285,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()> kv_total_blocks: 100, num_running_reqs: 0, num_waiting_reqs: 2, + ..Default::default() }, )?; wait_for_metrics( @@ -292,6 +296,11 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()> ("vllm:num_requests_waiting", "1", 2.0), ("vllm:kv_cache_usage_perc", "0", 0.25), ("vllm:kv_cache_usage_perc", "1", 0.5), + // Prefix-cache counters must surface through the upstream + // `vllm:prefix_cache_queries` / `vllm:prefix_cache_hits` gauges + // once the bridge wires LoadSnapshot through to SchedulerStats. + ("vllm:prefix_cache_queries", "0", 100.0), + ("vllm:prefix_cache_hits", "0", 80.0), ], &server.model_name, ) @@ -304,6 +313,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()> kv_total_blocks: 100, num_running_reqs: 3, num_waiting_reqs: 4, + ..Default::default() }, )?; server.publish_load( @@ -313,6 +323,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()> kv_total_blocks: 100, num_running_reqs: 5, num_waiting_reqs: 6, + ..Default::default() }, )?; wait_for_metrics( diff --git a/openinfer-vllm-frontend/src/bridge.rs b/openinfer-vllm-frontend/src/bridge.rs index 313d68ec9..404471e62 100644 --- a/openinfer-vllm-frontend/src/bridge.rs +++ b/openinfer-vllm-frontend/src/bridge.rs @@ -44,7 +44,9 @@ use vllm_engine_core_client::protocol::output::StopReason; use vllm_engine_core_client::protocol::output::UtilityCallOutput; use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::request::EngineCoreRequestType; +use vllm_engine_core_client::protocol::stats::BaseCacheStats; use vllm_engine_core_client::protocol::stats::PrefillStats; +use vllm_engine_core_client::protocol::stats::PrefixCacheStats; use vllm_engine_core_client::protocol::stats::SchedulerStats; use vllm_engine_core_client::protocol::utility::UtilityCallId; use vllm_engine_core_client::protocol::utility::UtilityOutput; @@ -681,6 +683,18 @@ async fn publish_scheduler_stats( } else { snapshot.kv_used_blocks as f64 / snapshot.kv_total_blocks as f64 }, + // Wire prefix-cache counters through to the upstream + // `prefix_cache_queries_total` / `prefix_cache_hits_total` gauges. + // Schedulers that do not yet track these leave them at 0, so the + // pipeline stays correct (metrics read 0) rather than silent. + prefix_cache_stats: PrefixCacheStats { + base: BaseCacheStats { + queries: snapshot.prefix_cache_queries, + hits: snapshot.prefix_cache_hits, + ..BaseCacheStats::default() + }, + ..PrefixCacheStats::default() + }, ..SchedulerStats::default() }; let outputs = RequestBatchOutputs { diff --git a/openinfer-vllm-frontend/src/bridge/tests.rs b/openinfer-vllm-frontend/src/bridge/tests.rs index f376359cd..b47624613 100644 --- a/openinfer-vllm-frontend/src/bridge/tests.rs +++ b/openinfer-vllm-frontend/src/bridge/tests.rs @@ -570,6 +570,8 @@ async fn load_snapshots_become_stats_only_batches() { kv_total_blocks: 100, num_running_reqs: 2, num_waiting_reqs: 1, + prefix_cache_queries: 80, + prefix_cache_hits: 64, }); let (output_tx, mut output_rx) = mpsc::unbounded_channel(); let shutdown = CancellationToken::new(); @@ -591,6 +593,11 @@ async fn load_snapshots_become_stats_only_batches() { assert_eq!(stats.num_running_reqs, 2); assert_eq!(stats.num_waiting_reqs, 1); assert!((stats.kv_cache_usage - 0.25).abs() < 1e-9); + // Prefix-cache counters must be wired through to SchedulerStats so the + // upstream `prefix_cache_queries_total` / `prefix_cache_hits_total` + // gauges reflect real hits instead of staying 0. + assert_eq!(stats.prefix_cache_stats.base.queries, 80); + assert_eq!(stats.prefix_cache_stats.base.hits, 64); load_tx.send_replace(LoadSnapshot::default()); let batch = match output_rx.recv().await.expect("drained stats batch") {