Draft-model fallback for N-gram proposer misses (ngram_fallback = "draft")

crates/mesh-llm-config/src/model.rs

@@ -603,6 +603,8 @@ pub struct SpeculativeConfig {
     pub verify_window_min_tokens: Option<u32>,
     pub verify_window_max_tokens: Option<u32>,
     pub verify_window_pipeline_depth: Option<u32>,
+    pub verify_window_runahead_tokens: Option<u32>,
+    pub ngram_fallback: Option<String>,
     pub spec_default: Option<BoolOrAuto>,
     pub(crate) legacy_draft_model_path_used: bool,
 }
@@ -654,6 +656,8 @@ impl SpeculativeConfig {
             verify_window_min_tokens: pick!(verify_window_min_tokens),
             verify_window_max_tokens: pick!(verify_window_max_tokens),
             verify_window_pipeline_depth: pick!(verify_window_pipeline_depth),
+            verify_window_runahead_tokens: pick!(verify_window_runahead_tokens),
+            ngram_fallback: pick!(ngram_fallback),
             spec_default: pick!(spec_default),
             legacy_draft_model_path_used: overrides
                 .filter(|config| config.draft_model.is_some())
@@ -727,6 +731,10 @@ struct SpeculativeConfigRaw {
     #[serde(default)]
     verify_window_pipeline_depth: Option<u32>,
     #[serde(default)]
+    verify_window_runahead_tokens: Option<u32>,
+    #[serde(default)]
+    ngram_fallback: Option<String>,
+    #[serde(default)]
     spec_default: Option<BoolOrAuto>,
 }
 
@@ -771,6 +779,8 @@ impl<'de> Deserialize<'de> for SpeculativeConfig {
             verify_window_min_tokens: raw.verify_window_min_tokens,
             verify_window_max_tokens: raw.verify_window_max_tokens,
             verify_window_pipeline_depth: raw.verify_window_pipeline_depth,
+            verify_window_runahead_tokens: raw.verify_window_runahead_tokens,
+            ngram_fallback: raw.ngram_fallback,
             spec_default: raw.spec_default,
             legacy_draft_model_path_used: legacy_used,
         })
@@ -833,6 +843,11 @@ impl Serialize for SpeculativeConfig {
             "verify_window_pipeline_depth",
             &self.verify_window_pipeline_depth,
         )?;
+        map.serialize_entry(
+            "verify_window_runahead_tokens",
+            &self.verify_window_runahead_tokens,
+        )?;
+        map.serialize_entry("ngram_fallback", &self.ngram_fallback)?;
         map.serialize_entry("spec_default", &self.spec_default)?;
         map.end()
     }

crates/mesh-llm-config/src/model/built_in_schema/control_behavior/speculative.rs

@@ -73,7 +73,9 @@ pub(super) fn apply_speculative_behavior(
         | "native_mtp_suppress_cooldown_draft_limit"
         | "verify_window_min_tokens"
         | "verify_window_max_tokens"
-        | "verify_window_pipeline_depth" => {}
+        | "verify_window_pipeline_depth"
+        | "verify_window_runahead_tokens"
+        | "ngram_fallback" => {}
         _ => {}
     }
 }

crates/mesh-llm-config/src/model/built_in_schema/declarations.rs

@@ -659,6 +659,11 @@ fn speculative_settings(prefix: &str) -> Vec<ConfigSettingSchema> {
             &format!("{prefix}.verify_window_pipeline_depth"),
             ConfigValueSchema::Integer,
         ),
+        basic_setting(
+            &format!("{prefix}.verify_window_runahead_tokens"),
+            ConfigValueSchema::Integer,
+        ),
+        basic_setting(&format!("{prefix}.ngram_fallback"), ConfigValueSchema::String),
         basic_setting(&format!("{prefix}.spec_default"), bool_or_auto_schema()),
     ]
 }

crates/mesh-llm-config/src/model_validation.rs

@@ -9,7 +9,7 @@ use crate::model::{
     SkippyConfig, SpeculativeConfig, StringOrStringList, merge_hardware, merge_model_fit,
     merge_multimodal, merge_throughput,
 };
-use skippy_protocol::MAX_VERIFY_WINDOW_PIPELINE_DEPTH;
+use skippy_protocol::{MAX_VERIFY_WINDOW_PIPELINE_DEPTH, MAX_VERIFY_WINDOW_RUNAHEAD_TOKENS};
 
 use crate::validation_support::{
     looks_like_model_identifier, validate_allowed, validate_bool_or_auto, validate_hf_pair,
@@ -633,7 +633,23 @@ fn validate_verify_window_controls(
         &format!("{base_path}.verify_window_pipeline_depth"),
         1,
         u32::try_from(MAX_VERIFY_WINDOW_PIPELINE_DEPTH).expect("verify depth limit fits u32"),
-    )
+    )?;
+    validate_optional_u32_range(
+        config.verify_window_runahead_tokens,
+        &format!("{base_path}.verify_window_runahead_tokens"),
+        // Zero is the documented fixed-depth sentinel, so a model-level block
+        // can switch run-ahead back off when the global defaults enable it.
+        0,
+        u32::try_from(MAX_VERIFY_WINDOW_RUNAHEAD_TOKENS).expect("runahead limit fits u32"),
+    )?;
+    if let Some(fallback) = config.ngram_fallback.as_deref() {
+        validate_allowed(
+            fallback,
+            &["draft", "none"],
+            &format!("{base_path}.ngram_fallback"),
+        )?;
+    }
+    Ok(())
 }
 
 fn validate_request_defaults(config: &RequestDefaultsConfig, base_path: &str) -> DiagnosticResult {

crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs

@@ -79,26 +79,36 @@ pub(crate) use stage::{
 pub(crate) use topology::{StageTopologyParticipant, plan_package_identity_topology};
 
 const BENCH_DOWNSTREAM_WIRE_DELAY_MS_ENV: &str = "MESH_LLM_BENCH_DOWNSTREAM_WIRE_DELAY_MS";
+const BENCH_DOWNSTREAM_WIRE_JITTER_MS_ENV: &str = "MESH_LLM_BENCH_DOWNSTREAM_WIRE_JITTER_MS";
+const BENCH_DOWNSTREAM_WIRE_STALL_MS_ENV: &str = "MESH_LLM_BENCH_DOWNSTREAM_WIRE_STALL_MS";
+const BENCH_DOWNSTREAM_WIRE_STALL_P_ENV: &str = "MESH_LLM_BENCH_DOWNSTREAM_WIRE_STALL_P";
 
 fn benchmark_downstream_wire_condition() -> Result<WireCondition> {
-    let delay_ms = match env::var(BENCH_DOWNSTREAM_WIRE_DELAY_MS_ENV) {
-        Ok(value) => parse_benchmark_downstream_wire_delay_ms(&value)?,
-        Err(env::VarError::NotPresent) => 0.0,
+    let delay_ms = parse_benchmark_wire_env(BENCH_DOWNSTREAM_WIRE_DELAY_MS_ENV)?;
+    let jitter_ms = parse_benchmark_wire_env(BENCH_DOWNSTREAM_WIRE_JITTER_MS_ENV)?;
+    let stall_ms = parse_benchmark_wire_env(BENCH_DOWNSTREAM_WIRE_STALL_MS_ENV)?;
+    let stall_p = parse_benchmark_wire_env(BENCH_DOWNSTREAM_WIRE_STALL_P_ENV)?;
+    WireCondition::with_jitter(delay_ms, None, jitter_ms, stall_ms, stall_p)
+}
+
+fn parse_benchmark_wire_env(name: &'static str) -> Result<f64> {
+    match env::var(name) {
+        Ok(value) => parse_benchmark_downstream_wire_value(name, &value),
+        Err(env::VarError::NotPresent) => Ok(0.0),
         Err(env::VarError::NotUnicode(_)) => {
-            anyhow::bail!("{BENCH_DOWNSTREAM_WIRE_DELAY_MS_ENV} must be valid UTF-8")
+            anyhow::bail!("{name} must be valid UTF-8")
         }
-    };
-    WireCondition::new(delay_ms, None)
+    }
 }
 
-fn parse_benchmark_downstream_wire_delay_ms(value: &str) -> Result<f64> {
-    let delay_ms = value.parse::<f64>().with_context(|| {
-        format!("{BENCH_DOWNSTREAM_WIRE_DELAY_MS_ENV} must be a finite non-negative number")
-    })?;
-    if !delay_ms.is_finite() || delay_ms < 0.0 {
-        anyhow::bail!("{BENCH_DOWNSTREAM_WIRE_DELAY_MS_ENV} must be a finite non-negative number");
+fn parse_benchmark_downstream_wire_value(name: &str, value: &str) -> Result<f64> {
+    let parsed = value
+        .parse::<f64>()
+        .with_context(|| format!("{name} must be a finite non-negative number"))?;
+    if !parsed.is_finite() || parsed < 0.0 {
+        anyhow::bail!("{name} must be a finite non-negative number");
     }
-    Ok(delay_ms)
+    Ok(parsed)
 }
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -1345,17 +1355,20 @@ mod tests {
 
     #[test]
     fn benchmark_wire_delay_accepts_finite_non_negative_values() {
-        assert_eq!(parse_benchmark_downstream_wire_delay_ms("0").unwrap(), 0.0);
         assert_eq!(
-            parse_benchmark_downstream_wire_delay_ms("25.5").unwrap(),
+            parse_benchmark_downstream_wire_value("test", "0").unwrap(),
+            0.0
+        );
+        assert_eq!(
+            parse_benchmark_downstream_wire_value("test", "25.5").unwrap(),
             25.5
         );
     }
 
     #[test]
     fn benchmark_wire_delay_rejects_invalid_values() {
         for value in ["-1", "NaN", "inf", "not-a-number"] {
-            assert!(parse_benchmark_downstream_wire_delay_ms(value).is_err());
+            assert!(parse_benchmark_downstream_wire_value("test", value).is_err());
         }
     }
 

crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs

@@ -136,6 +136,7 @@ pub(super) fn resolve_speculative_config(
         model_config,
         global_config,
         package_generation,
+        has_draft_model: draft_model_path.is_some(),
     })?;
     // A standalone N-gram plan (no native MTP, no draft model) runs in the
     // legacy `ngram` mode so the embedded frontend derives its window from the
@@ -201,6 +202,7 @@ struct DecodeResolutionInput<'a> {
     model_config: Option<&'a SpeculativeConfig>,
     global_config: Option<&'a SpeculativeConfig>,
     package_generation: Option<&'a PackageGenerationInfo>,
+    has_draft_model: bool,
 }
 
 #[allow(clippy::too_many_lines)]
@@ -397,9 +399,53 @@ fn resolve_decode_config(input: DecodeResolutionInput<'_>) -> Result<Speculative
             .and_then(|config| config.verify_window_pipeline_depth),
     )
     .map_or(config.verify_window.pipeline_depth, |value| value as usize);
+    config.verify_window.runahead_max_tokens = pick_optional_u32(
+        input
+            .model_config
+            .and_then(|config| config.verify_window_runahead_tokens),
+        input
+            .global_config
+            .and_then(|config| config.verify_window_runahead_tokens),
+    )
+    .map_or(config.verify_window.runahead_max_tokens, |value| {
+        value as usize
+    });
     if config.verify_window.min_tokens > config.verify_window.max_tokens {
         bail!("skippy speculative verify window requires min_tokens <= max_tokens");
     }
+    let ngram_fallback = pick_string(
+        input
+            .model_config
+            .and_then(|value| value.ngram_fallback.as_deref()),
+        input
+            .global_config
+            .and_then(|value| value.ngram_fallback.as_deref()),
+        None,
+    );
+    config.ngram_fallback_draft = match ngram_fallback {
+        "draft" => {
+            if config.ngram.is_none() {
+                bail!("skippy speculative ngram_fallback = \"draft\" requires an N-gram strategy");
+            }
+            // Both of these would otherwise start cleanly and never take the
+            // fallback path: the operator gets baseline behaviour and a
+            // telemetry counter stuck at zero, indistinguishable from a
+            // proposer that simply never missed.
+            if !input.has_draft_model {
+                bail!(
+                    "skippy speculative ngram_fallback = \"draft\" requires speculative.draft_model"
+                );
+            }
+            if config.verify_window.pipeline_depth <= 1 {
+                bail!(
+                    "skippy speculative ngram_fallback = \"draft\" requires verify_window_pipeline_depth > 1; the classic serial draft loop is authoritative at depth 1"
+                );
+            }
+            true
+        }
+        "none" | "" => false,
+        other => bail!("skippy speculative ngram_fallback must be draft or none, got {other}"),
+    };
     config.validate()?;
     Ok(config)
 }
@@ -479,6 +525,7 @@ fn package_decode_config(
             min_tokens: 1,
             max_tokens: 4,
             pipeline_depth: 1,
+            runahead_max_tokens: 0,
         });
     let effective_strategy = match (native_mtp.enabled, ngram.as_ref().map(|value| value.kind)) {
         (true, Some(NgramProposerKind::Cache)) => "native-mtp+ngram-cache",
@@ -494,6 +541,7 @@ fn package_decode_config(
         ngram,
         extension,
         verify_window,
+        ngram_fallback_draft: false,
     }))
 }
 
@@ -564,6 +612,7 @@ fn verify_window_config(policy: &PackageWindowPolicyInfo) -> VerifyWindowConfig
         min_tokens: policy.min_window as usize,
         max_tokens: policy.max_window as usize,
         pipeline_depth: policy.pipeline_depth.unwrap_or(1) as usize,
+        runahead_max_tokens: 0,
     }
 }
 

crates/mesh-llm-host-runtime/src/inference/skippy/resolver/tests.rs

@@ -1592,3 +1592,110 @@ verify_window_pipeline_depth = 2
     assert_eq!(translated.max_proposal_tokens, 48);
     assert_eq!(args.speculative.verify_window.pipeline_depth, 2);
 }
+
+#[test]
+fn model_level_zero_runahead_overrides_a_positive_global_default() {
+    use crate::plugin::SpeculativeConfig;
+    let global: SpeculativeConfig = toml::from_str(
+        r#"
+strategy = "ngram-suffix"
+ngram_proposer = "suffix"
+ngram_min = 5
+ngram_max = 32
+verify_window_pipeline_depth = 2
+verify_window_runahead_tokens = 256
+"#,
+    )
+    .expect("parse global speculative config");
+    let model: SpeculativeConfig = toml::from_str(
+        r#"
+verify_window_runahead_tokens = 0
+"#,
+    )
+    .expect("parse model speculative config");
+    let inherited = super::speculative::resolve_speculative_config(
+        None,
+        Some(&global),
+        "meshllm/test-model",
+        std::path::Path::new("/nonexistent/test-model.gguf"),
+        None,
+    )
+    .expect("global run-ahead must resolve");
+    assert_eq!(inherited.decode.verify_window.runahead_max_tokens, 256);
+    let overridden = super::speculative::resolve_speculative_config(
+        Some(&model),
+        Some(&global),
+        "meshllm/test-model",
+        std::path::Path::new("/nonexistent/test-model.gguf"),
+        None,
+    )
+    .expect("model-level zero must resolve to fixed-depth mode");
+    assert_eq!(
+        overridden.decode.verify_window.runahead_max_tokens, 0,
+        "Some(0) at the model level must win over the inherited positive default"
+    );
+}
+
+#[test]
+fn draft_fallback_without_a_draft_model_is_rejected_not_silently_ignored() {
+    use crate::plugin::SpeculativeConfig;
+    let config: SpeculativeConfig = toml::from_str(
+        r#"
+strategy = "ngram-suffix"
+ngram_proposer = "suffix"
+ngram_min = 5
+ngram_max = 32
+verify_window_pipeline_depth = 2
+ngram_fallback = "draft"
+"#,
+    )
+    .expect("parse speculative config");
+    let error = super::speculative::resolve_speculative_config(
+        Some(&config),
+        None,
+        "meshllm/test-model",
+        std::path::Path::new("/nonexistent/test-model.gguf"),
+        None,
+    )
+    .expect_err("a draft fallback with no draft model must not resolve");
+
+    assert!(
+        error
+            .to_string()
+            .contains("requires speculative.draft_model"),
+        "unexpected error: {error}"
+    );
+}
+
+#[test]
+fn draft_fallback_at_pipeline_depth_one_is_rejected() {
+    use crate::plugin::SpeculativeConfig;
+    let config: SpeculativeConfig = toml::from_str(
+        r#"
+strategy = "ngram-suffix"
+ngram_proposer = "suffix"
+ngram_min = 5
+ngram_max = 32
+verify_window_pipeline_depth = 1
+ngram_fallback = "draft"
+draft_model = "meshllm/draft-model"
+draft_max_tokens = 4
+"#,
+    )
+    .expect("parse speculative config");
+    let error = super::speculative::resolve_speculative_config(
+        Some(&config),
+        None,
+        "meshllm/test-model",
+        std::path::Path::new("/nonexistent/test-model.gguf"),
+        None,
+    )
+    .expect_err("depth 1 keeps the serial draft loop authoritative");
+
+    assert!(
+        error
+            .to_string()
+            .contains("verify_window_pipeline_depth > 1"),
+        "unexpected error: {error}"
+    );
+}