diff --git a/crates/mesh-llm-host-runtime/src/runtime/local.rs b/crates/mesh-llm-host-runtime/src/runtime/local.rs index 6ea56ad26c..3bcc56069c 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local.rs @@ -228,6 +228,7 @@ pub(super) struct LocalRuntimeModelStartSpec<'a> { pub(super) mmproj_override: Option<&'a Path>, pub(super) ctx_size_override: Option, pub(super) pinned_gpu: Option<&'a crate::runtime::StartupPinnedGpuTarget>, + pub(super) device_override: Option, pub(super) capacity_budget_bytes: Option, pub(super) cache_type_k_override: Option<&'a str>, pub(super) cache_type_v_override: Option<&'a str>, @@ -250,6 +251,7 @@ pub(super) struct LocalOpenAiModelStartSpec<'a> { pub(super) mmproj_override: Option<&'a Path>, pub(super) ctx_size_override: Option, pub(super) pinned_gpu: Option<&'a crate::runtime::StartupPinnedGpuTarget>, + pub(super) device_override: Option, pub(super) capacity_budget_bytes: u64, pub(super) cache_type_k_override: Option<&'a str>, pub(super) cache_type_v_override: Option<&'a str>, @@ -350,6 +352,9 @@ pub(super) fn resolve_local_openai_skippy_config( if let Some(gpu) = spec.pinned_gpu { resolved.hardware.device = Some(gpu.backend_device.clone()); } + if let Some(device) = &spec.device_override { + resolved.hardware.device = Some(device.clone()); + } Ok(resolved) } @@ -564,6 +569,7 @@ pub(super) async fn start_runtime_local_model( mmproj_override: spec.mmproj_override, ctx_size_override: spec.ctx_size_override, pinned_gpu: spec.pinned_gpu, + device_override: spec.device_override, capacity_budget_bytes: local_capacity_bytes, cache_type_k_override: spec.cache_type_k_override, cache_type_v_override: spec.cache_type_v_override, @@ -738,7 +744,9 @@ async fn start_local_skippy_model( .with_openai_guardrails(skippy::skippy_openai_guardrails_for_policy_handle( spec.openai_guardrail_policy.clone(), )); - if let Some(gpu) = spec.pinned_gpu { + if spec.device_override.is_none() + && let Some(gpu) = spec.pinned_gpu + { options = options.with_selected_device(pinned_skippy_device(gpu)); } let _ = emit_event(OutputEvent::ModelLoading { @@ -852,7 +860,9 @@ async fn start_local_layer_package_model( runtime_options.config.ctx_size = context_length; runtime_options.config.lane_count = plan.slots as u32; runtime_options.config.filter_tensors_on_load = true; - if let Some(gpu) = spec.pinned_gpu { + if spec.device_override.is_none() + && let Some(gpu) = spec.pinned_gpu + { runtime_options.config.selected_device = Some(pinned_stage_device(gpu)); } runtime_options.config.load_mode = LoadMode::LayerPackage; diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_model_only.rs b/crates/mesh-llm-host-runtime/src/runtime/local_model_only.rs index b4ab490756..d5fde80cff 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_model_only.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_model_only.rs @@ -3,9 +3,10 @@ use super::{ SkippyNativeLogForwardingGuard, acquire_instance_runtime, apply_runtime_cli_speculative_overrides, apply_runtime_config_options, build_startup_model_specs, cleanup_run_auto_runtime_dir, configure_run_auto_process_state, - emit_shutdown, openai_guardrail_policy_handle, preflight_config_owned_startup_models, + emit_shutdown, openai_guardrail_policy_handle, preflight_pinned_startup_models, resolve_local_model_only_startup_models, runtime_model_required_bytes, - skippy_telemetry_options, start_local_openai_model, wait_shutdown_signal, + skippy_telemetry_options, start_local_openai_model, startup_device_override, + wait_shutdown_signal, }; use crate::inference::election; use crate::plugin; @@ -119,7 +120,7 @@ pub(super) async fn run_local_model_only(mut options: RuntimeOptions) -> Result< "--local-model-only requires exactly one startup model" ); let mut startup_models = resolve_local_model_only_startup_models(&startup_specs).await?; - preflight_config_owned_startup_models( + preflight_pinned_startup_models( &config, &startup_specs, &mut startup_models, @@ -172,6 +173,7 @@ pub(super) async fn run_local_model_only(mut options: RuntimeOptions) -> Result< mmproj_override: model.mmproj_path.as_deref(), ctx_size_override: model.ctx_size, pinned_gpu: model.pinned_gpu.as_ref(), + device_override: startup_device_override(model.gpu_id.as_deref()), capacity_budget_bytes: local_capacity_bytes, cache_type_k_override: model.cache_type_k.as_deref(), cache_type_v_override: model.cache_type_v.as_deref(), diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split.rs index 4e6c5f683c..a408200c97 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split.rs @@ -240,6 +240,7 @@ pub(super) async fn start_runtime_split_model( flash_attention_override: spec.flash_attention_override, openai_guardrail_policy: spec.openai_guardrail_policy.clone(), pinned_gpu: spec.pinned_gpu, + device_override: spec.device_override.as_deref(), slots, skippy_telemetry: spec.skippy_telemetry.clone(), survey_telemetry: spec.survey_telemetry.clone(), @@ -275,6 +276,7 @@ pub(super) async fn start_runtime_split_model( flash_attention_override: spec.flash_attention_override, openai_guardrail_policy: spec.openai_guardrail_policy.clone(), pinned_gpu: spec.pinned_gpu.cloned(), + device_override: spec.device_override.clone(), slots, skippy_telemetry: spec.skippy_telemetry.clone(), survey_telemetry: spec.survey_telemetry.clone(), diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs index f4cd660e90..ec0549c015 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs @@ -85,6 +85,7 @@ pub(super) struct SplitTopologyCoordinator { pub(super) flash_attention_override: FlashAttentionType, pub(super) openai_guardrail_policy: OpenAiGuardrailPolicyHandle, pub(super) pinned_gpu: Option, + pub(super) device_override: Option, pub(super) slots: usize, pub(super) skippy_telemetry: skippy::SkippyTelemetryOptions, pub(super) survey_telemetry: survey::SurveyTelemetry, @@ -642,6 +643,7 @@ impl SplitTopologyCoordinator { flash_attention_override: self.flash_attention_override, openai_guardrail_policy: self.openai_guardrail_policy.clone(), pinned_gpu: self.pinned_gpu.as_ref(), + device_override: self.device_override.as_deref(), slots: self.slots, skippy_telemetry: self.skippy_telemetry.clone(), survey_telemetry: self.survey_telemetry.clone(), diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs index 240d9dc0c3..1362452d64 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs @@ -59,6 +59,7 @@ pub(super) struct SplitGenerationLoadSpec<'a> { pub(super) ctx_size: u32, pub(super) compact_meta: &'a models::gguf::GgufCompactMeta, pub(super) pinned_gpu: Option<&'a crate::runtime::StartupPinnedGpuTarget>, + pub(super) device_override: Option<&'a str>, pub(super) slots: usize, pub(super) cache_type_k_override: Option<&'a str>, pub(super) cache_type_v_override: Option<&'a str>, @@ -245,9 +246,11 @@ pub(super) async fn load_split_runtime_generation_inner( runtime_options.config.ctx_size = spec.ctx_size; runtime_options.config.lane_count = spec.slots as u32; runtime_options.config.filter_tensors_on_load = true; - if let Some(gpu) = spec.pinned_gpu { - runtime_options.config.selected_device = Some(pinned_stage_device(gpu)); - } + apply_split_generation_pinned_device( + &mut runtime_options.config, + spec.pinned_gpu, + spec.device_override, + ); runtime_options.config.load_mode = settings.load_mode.clone(); runtime_options.config.bind_addr = stage0_return_endpoint; runtime_options.config.upstream = None; @@ -600,6 +603,9 @@ pub(super) async fn split_generation_load_settings<'a>( if let Some(gpu) = spec.pinned_gpu { resolved.hardware.device = Some(gpu.backend_device.clone()); } + if let Some(device) = spec.device_override { + resolved.hardware.device = Some(device.to_string()); + } let embedded_openai = resolved.to_embedded_openai_args(activation_width, true)?; let lifecycle = configured_stage_lifecycle_intervals(spec.mesh_config, spec.config_model_id); let runtime_options = resolved.to_embedded_runtime_options( @@ -624,6 +630,18 @@ pub(super) async fn split_generation_load_settings<'a>( }) } +pub(super) fn apply_split_generation_pinned_device( + config: &mut skippy_protocol::StageConfig, + pinned_gpu: Option<&crate::runtime::StartupPinnedGpuTarget>, + device_override: Option<&str>, +) { + if device_override.is_none() + && let Some(gpu) = pinned_gpu + { + config.selected_device = Some(pinned_stage_device(gpu)); + } +} + pub(super) fn split_generation_load_mode(package: &skippy::SkippyPackageIdentity) -> LoadMode { if skippy::is_layer_package_ref(&package.package_ref) { LoadMode::LayerPackage diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs index 9aea25ce09..35adab70ff 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs @@ -88,6 +88,36 @@ pub(super) fn stage_load_request(load_mode: LoadMode) -> skippy::StageLoadReques } } +#[test] +fn split_generation_cli_device_override_survives_pinned_stage_selection() { + let mut config = skippy_protocol::StageConfig { + selected_device: Some(skippy_protocol::StageDevice { + backend_device: "CPU".to_string(), + stable_id: None, + index: None, + vram_bytes: None, + }), + ..Default::default() + }; + let pinned_gpu = crate::runtime::StartupPinnedGpuTarget { + index: 0, + stable_id: "pci:0000:65:00.0".to_string(), + backend_device: "CUDA0".to_string(), + vram_bytes: 24_000_000_000, + reserved_bytes: None, + }; + + apply_split_generation_pinned_device(&mut config, Some(&pinned_gpu), Some("CPU")); + + assert_eq!( + config + .selected_device + .as_ref() + .map(|device| device.backend_device.as_str()), + Some("CPU") + ); +} + pub(super) fn split_test_peer( seed: u8, model_name: &str, @@ -427,6 +457,7 @@ stop = ["END"] ctx_size: 8192, compact_meta: &compact_meta, pinned_gpu: None, + device_override: None, slots: 4, cache_type_k_override: None, cache_type_v_override: None, @@ -554,6 +585,7 @@ async fn split_stage_load_guards_family_kv_default_with_planned_metadata() { ctx_size: 4096, compact_meta: &incompatible_meta, pinned_gpu: None, + device_override: None, slots: 1, cache_type_k_override: None, cache_type_v_override: None, @@ -601,7 +633,7 @@ async fn split_stage_load_guards_family_kv_default_with_planned_metadata() { } #[tokio::test] -async fn runtime_resolver_uses_config_model_id_but_preserves_served_model_id() { +async fn runtime_resolver_uses_config_identity_and_honors_device_override() { let node = mesh::Node::new_for_tests(NodeRole::Host { http_port: 9337 }) .await .unwrap(); @@ -615,6 +647,7 @@ model = "other/model-ref" [models.hardware] model_path = "{model_path}" +device = "CUDA1" [models.throughput] threads = 17 @@ -645,6 +678,7 @@ max_tokens = 222 mmproj_override: None, ctx_size_override: None, pinned_gpu: None, + device_override: Some("CPU".to_string()), capacity_budget_bytes: node.vram_bytes(), cache_type_k_override: None, cache_type_v_override: None, @@ -680,6 +714,55 @@ max_tokens = 222 assert_eq!(resolved.request_defaults.max_tokens, 222); assert_eq!(resolved.model_fit.ctx_size, 4096); assert_eq!(resolved.throughput.parallel, 3); + assert_eq!(resolved.hardware.device.as_deref(), Some("CPU")); + // An explicit CLI artifact may use the same served name as a configured + // model, but must not inherit that entry's path or runtime tuning. + let cli_model_path = temp_dir.path().join("cli-selected.gguf"); + write_fake_gguf_model(&cli_model_path); + let cli_model_bytes = fs::metadata(&cli_model_path).unwrap().len(); + let cli_spec = LocalOpenAiModelStartSpec { + mesh_config: &mesh_config, + config_model_id: None, + model_path: &cli_model_path, + model_bytes: cli_model_bytes, + mmproj_override: None, + ctx_size_override: None, + pinned_gpu: None, + device_override: None, + capacity_budget_bytes: node.vram_bytes(), + cache_type_k_override: None, + cache_type_v_override: None, + n_batch_override: None, + n_ubatch_override: None, + flash_attention_override: FlashAttentionType::Auto, + parallel_override: None, + planning_profile: RuntimeResourcePlanningProfile::DedicatedLocal, + openai_guardrail_policy: openai_guardrail_policy_handle( + openai_frontend::GuardrailMode::Disabled, + ), + skippy_telemetry: skippy::SkippyTelemetryOptions::off(), + survey_telemetry: survey::SurveyTelemetry::disabled(), + hook_policy: None, + serving_hooks_factory: None, + http_bind_addr: "127.0.0.1:0".parse().expect("valid loopback address"), + }; + let cli_resolved = resolve_local_openai_skippy_config( + &cli_spec, + "configured/model-ref", + cli_model_bytes, + 4096, + 3, + None, + None, + ) + .expect("explicit CLI runtime config should not consult model entries"); + assert_eq!(cli_resolved.hardware.resolved_model_path, cli_model_path); + assert_eq!(cli_resolved.throughput.threads, None); + assert_eq!(cli_resolved.throughput.threads_batch, None); + assert_eq!( + cli_resolved.request_defaults.max_tokens, + skippy_server::CONTEXT_BUDGET_MAX_TOKENS + ); } #[test] diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs index e68dbc0d40..6c4ff6c110 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs @@ -1311,6 +1311,7 @@ async fn load_split_runtime_generation_stops_candidate_stages_after_partial_load ctx_size: 4096, compact_meta: &compact_meta, pinned_gpu: None, + device_override: None, slots: 1, cache_type_k_override: None, cache_type_v_override: None, diff --git a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs index 4181ebb7a1..f94b638ef0 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs @@ -139,6 +139,7 @@ pub(crate) async fn run_auto_load_runtime_model( mmproj_override: None, ctx_size_override, pinned_gpu: None, + device_override: None, capacity_budget_bytes: Some(capacity_budget_bytes), cache_type_k_override: model_overrides.and_then(|m| m.cache_type_k.as_deref()), cache_type_v_override: model_overrides.and_then(|m| m.cache_type_v.as_deref()), diff --git a/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs b/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs index 313059b803..922f7fcc96 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs @@ -23,7 +23,8 @@ use super::{ runtime_data_producer_for_console, runtime_startup_requirements, setup_run_auto_console_state, setup_run_auto_serving_surface, spawn_embedded_runtime_control_forwarder, spawn_run_auto_additional_model_tasks, spawn_run_auto_discovery_publisher, - start_run_auto_bootstrap_proxy, startup_local_model_loop, swarm_capture_observer_requested, + start_run_auto_bootstrap_proxy, startup_device_override, startup_local_model_loop, + swarm_capture_observer_requested, }; use crate::api; use crate::inference::{election, skippy}; @@ -1125,6 +1126,8 @@ pub(super) async fn spawn_run_auto_startup_model_tasks(ctx: RunAutoStartupTasksC let primary_mmproj = primary_startup_model.and_then(|model| model.mmproj_path.clone()); let primary_ctx_size = primary_startup_model.and_then(|model| model.ctx_size); let primary_pinned_gpu = primary_startup_model.and_then(|model| model.pinned_gpu.clone()); + let primary_device_override = + primary_startup_model.and_then(|model| startup_device_override(model.gpu_id.as_deref())); let primary_cache_type_k = primary_startup_model.and_then(|model| model.cache_type_k.clone()); let primary_cache_type_v = primary_startup_model.and_then(|model| model.cache_type_v.clone()); let primary_n_batch = primary_startup_model.and_then(|model| model.n_batch); @@ -1162,6 +1165,7 @@ pub(super) async fn spawn_run_auto_startup_model_tasks(ctx: RunAutoStartupTasksC mmproj_path: primary_mmproj, ctx_size: primary_ctx_size, pinned_gpu: primary_pinned_gpu, + device_override: primary_device_override, runtime_capacity_ledger: runtime_capacity_ledger.clone(), cache_type_k: primary_cache_type_k, cache_type_v: primary_cache_type_v, diff --git a/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs b/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs index 3f02d0e46f..61d4ad236b 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs @@ -1156,6 +1156,9 @@ pub(super) async fn spawn_run_auto_additional_model_tasks(ctx: RunAutoAdditional mmproj_path: extra_model.mmproj_path.clone(), ctx_size: extra_model.ctx_size, pinned_gpu: extra_model.pinned_gpu.clone(), + device_override: super::startup_models::startup_device_override( + extra_model.gpu_id.as_deref(), + ), runtime_capacity_ledger: ctx.runtime_capacity_ledger.clone(), cache_type_k: extra_model.cache_type_k.clone(), cache_type_v: extra_model.cache_type_v.clone(), diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs index 17c3f60d48..ddee2e53e3 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs @@ -70,6 +70,7 @@ pub(super) struct StartupLocalModelTask { pub(super) mmproj_path: Option, pub(super) ctx_size: Option, pub(super) pinned_gpu: Option, + pub(super) device_override: Option, pub(super) runtime_capacity_ledger: RuntimeCapacityLedger, pub(super) cache_type_k: Option, pub(super) cache_type_v: Option, @@ -576,6 +577,7 @@ pub(super) async fn startup_launch_runtime( mmproj_path, ctx_size, pinned_gpu, + device_override, runtime_capacity_ledger, cache_type_k, cache_type_v, @@ -605,6 +607,7 @@ pub(super) async fn startup_launch_runtime( mmproj_override: mmproj_path.map(PathBuf::as_path), ctx_size_override: ctx_size, pinned_gpu, + device_override: device_override.clone(), capacity_budget_bytes: None, cache_type_k_override: cache_type_k, cache_type_v_override: cache_type_v, diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_handles/startup_loop.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_handles/startup_loop.rs index efdd26a4f7..6f8bfb60df 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_handles/startup_loop.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_handles/startup_loop.rs @@ -15,6 +15,7 @@ pub(in crate::runtime) struct StartupLoopContext<'a> { pub(super) mmproj_path: Option<&'a PathBuf>, pub(super) ctx_size: Option, pub(super) pinned_gpu: Option<&'a StartupPinnedGpuTarget>, + pub(super) device_override: Option<&'a str>, pub(super) runtime_capacity_ledger: &'a RuntimeCapacityLedger, pub(super) cache_type_k: Option<&'a str>, pub(super) cache_type_v: Option<&'a str>, @@ -109,6 +110,7 @@ pub(in crate::runtime) struct StartupLaunchRuntimeContext<'a> { pub(super) mmproj_path: Option<&'a PathBuf>, pub(super) ctx_size: Option, pub(super) pinned_gpu: Option<&'a StartupPinnedGpuTarget>, + pub(super) device_override: Option, pub(super) runtime_capacity_ledger: &'a RuntimeCapacityLedger, pub(super) cache_type_k: Option<&'a str>, pub(super) cache_type_v: Option<&'a str>, @@ -244,6 +246,7 @@ pub(in crate::runtime) async fn startup_handle_local_fallback_event( mmproj_override: ctx.mmproj_path.map(PathBuf::as_path), ctx_size_override: ctx.ctx_size, pinned_gpu: ctx.pinned_gpu, + device_override: ctx.device_override.map(str::to_string), capacity_budget_bytes: Some(reservation.capacity_budget_bytes()), cache_type_k_override: ctx.cache_type_k, cache_type_v_override: ctx.cache_type_v, @@ -670,6 +673,7 @@ async fn launch_startup_local_model_task( mmproj_path: params.mmproj_path.as_ref(), ctx_size: params.ctx_size, pinned_gpu: params.pinned_gpu.as_ref(), + device_override: params.device_override.clone(), runtime_capacity_ledger: ¶ms.runtime_capacity_ledger, cache_type_k: params.cache_type_k.as_deref(), cache_type_v: params.cache_type_v.as_deref(), @@ -712,6 +716,7 @@ fn startup_loop_context<'a>( mmproj_path: params.mmproj_path.as_ref(), ctx_size: params.ctx_size, pinned_gpu: params.pinned_gpu.as_ref(), + device_override: params.device_override.as_deref(), runtime_capacity_ledger: ¶ms.runtime_capacity_ledger, cache_type_k: params.cache_type_k.as_deref(), cache_type_v: params.cache_type_v.as_deref(), diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs index e5356a605d..e03762761f 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs @@ -33,7 +33,14 @@ pub(super) struct StartupModelSpec { pub(super) mmproj_ref: Option, pub(super) ctx_size: Option, pub(super) gpu_id: Option, - pub(super) config_owned: bool, + /// Whether `gpu_id` came from an explicit non-auto CLI `--device`. + pub(super) cli_device_override: bool, + /// Whether pinned startup preflight must resolve this model's GPU selector. + /// + /// Config-owned models opt in. An explicit CLI `--model` also opts in + /// when its ref exactly and uniquely matches a configured model. Ad-hoc + /// model refs and GGUF paths remain outside per-model config ownership. + pub(super) resolve_pinned_gpu: bool, pub(super) parallel: Option, pub(super) cache_type_k: Option, pub(super) cache_type_v: Option, @@ -458,7 +465,7 @@ pub(super) async fn resolve_eager_startup_models( startup_specs: &[StartupModelSpec], ) -> Result> { let mut startup_models = resolve_startup_models(startup_specs, options.split).await?; - preflight_config_owned_startup_models( + preflight_pinned_startup_models( config, startup_specs, &mut startup_models, @@ -498,6 +505,7 @@ pub(super) fn runtime_options_for_test(args: &[&str]) -> RuntimeOptions { "--model" => options.model.push(next_test_arg(&mut iter, arg).into()), "--gguf" => options.gguf.push(next_test_arg(&mut iter, arg).into()), "--mmproj" => options.mmproj = Some(next_test_arg(&mut iter, arg).into()), + "--device" => options.device = Some(next_test_arg(&mut iter, arg).to_string()), "--ctx-size" => { options.ctx_size = Some( next_test_arg(&mut iter, arg) @@ -687,6 +695,73 @@ fn configured_server_alias( .map(str::to_string) } +/// Return the effective persisted device selector for one configured model. +fn configured_model_gpu_id( + config: &plugin::MeshConfig, + model: &plugin::ModelConfigEntry, +) -> Option { + model + .hardware + .as_ref() + .and_then(|hardware| hardware.device.clone()) + .or_else(|| model.gpu_id.clone()) + .or_else(|| configured_default_gpu_id(config)) +} + +/// Return the global device selector inherited by ad-hoc startup models. +fn configured_default_gpu_id(config: &plugin::MeshConfig) -> Option { + config + .defaults + .as_ref() + .and_then(|defaults| defaults.hardware.as_ref()) + .and_then(|hardware| hardware.device.clone()) +} + +/// Match only an exact CLI `--model` ref. +/// +/// The CLI has no profile selector, so duplicate refs are ambiguous. GGUF +/// paths and aliases do not participate in per-model config matching. +fn matching_config_model<'a>( + config: &'a plugin::MeshConfig, + model_ref: &Path, +) -> Result> { + let Some(model_ref) = model_ref.to_str() else { + return Ok(None); + }; + let mut matches = config + .models + .iter() + .filter(|model| model.model == model_ref); + let Some(first) = matches.next() else { + return Ok(None); + }; + if matches.next().is_some() { + anyhow::bail!( + "CLI --model '{}' matches multiple configured model entries; select a unique configured model ref", + model_ref + ); + } + Ok(Some(first)) +} + +/// Apply CLI device precedence without treating `--device auto` as a pin. +fn effective_startup_gpu_id(options: &RuntimeOptions, persisted: Option<&str>) -> Option { + options + .device + .as_deref() + .map(str::trim) + .filter(|device| !device.is_empty() && !device.eq_ignore_ascii_case("auto")) + .map(str::to_string) + .or_else(|| persisted.map(str::to_string)) +} + +fn has_explicit_startup_device(options: &RuntimeOptions) -> bool { + options.device.as_deref().is_some_and(|device| { + let device = device.trim(); + !device.is_empty() && !device.eq_ignore_ascii_case("auto") + }) +} + pub(super) fn build_startup_model_specs( options: &RuntimeOptions, config: &plugin::MeshConfig, @@ -716,8 +791,9 @@ pub(super) fn build_startup_model_specs( .clone() .or_else(|| effective.mmproj.as_ref().map(PathBuf::from)), ctx_size: options.ctx_size.or(effective.ctx_size), - gpu_id: effective.gpu_id.clone(), - config_owned: false, + gpu_id: effective_startup_gpu_id(options, effective.gpu_id.as_deref()), + cli_device_override: has_explicit_startup_device(options), + resolve_pinned_gpu: false, parallel: effective.parallel, cache_type_k: effective.cache_type_k.clone(), cache_type_v: effective.cache_type_v.clone(), @@ -743,8 +819,9 @@ pub(super) fn build_startup_model_specs( config_model_id: None, mmproj_ref: effective.mmproj.as_ref().map(PathBuf::from), ctx_size: options.ctx_size.or(effective.ctx_size), - gpu_id: effective.gpu_id.clone(), - config_owned: false, + gpu_id: effective_startup_gpu_id(options, effective.gpu_id.as_deref()), + cli_device_override: has_explicit_startup_device(options), + resolve_pinned_gpu: false, parallel: effective.parallel, cache_type_k: effective.cache_type_k.clone(), cache_type_v: effective.cache_type_v.clone(), @@ -757,16 +834,21 @@ pub(super) fn build_startup_model_specs( }); } for model in &options.model { + let matching_config = matching_config_model(config, model)?; let effective = effective_startup_model_config(&model.display().to_string(), None, defaults); + let persisted_gpu_id = matching_config + .and_then(|model| configured_model_gpu_id(config, model)) + .or_else(|| configured_default_gpu_id(config)); specs.push(StartupModelSpec { model_ref: model.clone(), declared_ref: None, config_model_id: None, mmproj_ref: effective.mmproj.as_ref().map(PathBuf::from), ctx_size: options.ctx_size.or(effective.ctx_size), - gpu_id: effective.gpu_id.clone(), - config_owned: false, + gpu_id: effective_startup_gpu_id(options, persisted_gpu_id.as_deref()), + cli_device_override: has_explicit_startup_device(options), + resolve_pinned_gpu: matching_config.is_some(), parallel: effective.parallel, cache_type_k: effective.cache_type_k.clone(), cache_type_v: effective.cache_type_v.clone(), @@ -840,8 +922,9 @@ pub(super) fn build_startup_model_specs( config_model_id: Some(model.model.clone()), mmproj_ref: effective.mmproj.as_ref().map(PathBuf::from), ctx_size: options.ctx_size.or(effective.ctx_size), - gpu_id: effective.gpu_id.clone(), - config_owned: true, + gpu_id: effective_startup_gpu_id(options, effective.gpu_id.as_deref()), + cli_device_override: has_explicit_startup_device(options), + resolve_pinned_gpu: true, parallel: effective.parallel, cache_type_k: effective.cache_type_k.clone(), cache_type_v: effective.cache_type_v.clone(), @@ -1037,14 +1120,16 @@ pub(super) fn resolve_split_layer_package(model_query: &str, model_path: &Path) models::remote_catalog::find_huggingface_layer_package(model_query) } -pub(super) fn preflight_config_owned_startup_models( +pub(super) fn preflight_pinned_startup_models( config: &plugin::MeshConfig, specs: &[StartupModelSpec], plans: &mut [StartupModelPlan], binary_flavor: Option, backend_probe: Option<&backend::BinaryBackendDeviceProbe>, ) -> Result<()> { - if config.gpu.assignment != plugin::GpuAssignment::Pinned { + if config.gpu.assignment != plugin::GpuAssignment::Pinned + && plans.iter().all(|plan| plan.gpu_id.is_none()) + { return Ok(()); } @@ -1053,13 +1138,7 @@ pub(super) fn preflight_config_owned_startup_models( .or(binary_flavor); let mut survey = hardware::query(pinned_startup_preflight_metrics()); apply_backend_devices_for_flavor(&mut survey.gpus, binary_flavor); - preflight_config_owned_startup_models_with_gpus( - config, - specs, - plans, - &survey.gpus, - backend_probe, - ) + preflight_pinned_startup_models_with_gpus(config, specs, plans, &survey.gpus, backend_probe) } pub(super) fn apply_backend_devices_for_flavor( @@ -1091,35 +1170,39 @@ pub(super) fn pinned_startup_preflight_metrics() -> &'static [hardware::Metric] ] } -pub(super) fn preflight_config_owned_startup_models_with_gpus( +pub(super) fn preflight_pinned_startup_models_with_gpus( config: &plugin::MeshConfig, specs: &[StartupModelSpec], plans: &mut [StartupModelPlan], gpus: &[hardware::GpuFacts], backend_probe: Option<&backend::BinaryBackendDeviceProbe>, ) -> Result<()> { - if config.gpu.assignment != plugin::GpuAssignment::Pinned { - return Ok(()); - } - anyhow::ensure!( specs.len() == plans.len(), "startup model preflight received mismatched specs/plans" ); for (spec, plan) in specs.iter().zip(plans.iter_mut()) { - if !spec.config_owned { + let must_resolve_device = + spec.resolve_pinned_gpu && config.gpu.assignment == plugin::GpuAssignment::Pinned; + if !must_resolve_device && plan.gpu_id.is_none() { + continue; + } + if is_cpu_startup_device(plan.gpu_id.as_deref()) { continue; } - let resolved_gpu = hardware::resolve_pinned_gpu_strict(plan.gpu_id.as_deref(), gpus) - .map_err(anyhow::Error::new) - .with_context(|| { - format!( - "startup model '{}' failed pinned GPU preflight", - plan.declared_ref - ) - })?; + let resolved_gpu = resolve_requested_startup_device( + plan.gpu_id.as_deref(), + gpus, + spec.cli_device_override, + ) + .with_context(|| { + format!( + "startup model '{}' failed pinned GPU preflight", + plan.declared_ref + ) + })?; let stable_id = resolved_gpu.stable_id.clone().ok_or_else(|| { anyhow::anyhow!( @@ -1174,6 +1257,72 @@ pub(super) fn preflight_config_owned_startup_models_with_gpus( Ok(()) } +/// Resolve a stable GPU ID or a concrete backend device name. +fn resolve_requested_startup_device<'a>( + requested_device: Option<&str>, + gpus: &'a [hardware::GpuFacts], + allow_backend_device_name: bool, +) -> Result<&'a hardware::GpuFacts> { + match hardware::resolve_pinned_gpu_strict(requested_device, gpus) { + Ok(gpu) => Ok(gpu), + Err(hardware::PinnedGpuResolverError::NonPinnableConfiguredId { + configured_id, .. + }) if allow_backend_device_name => { + resolve_startup_backend_device_by_name(&configured_id, gpus) + } + Err(err) => Err(anyhow::Error::new(err)), + } +} + +fn is_cpu_startup_device(requested_device: Option<&str>) -> bool { + requested_device.is_some_and(|device| device.trim().eq_ignore_ascii_case("CPU")) +} + +/// Preserve CPU for the runtime resolver while bypassing GPU-only preflight. +pub(super) fn startup_device_override(requested_device: Option<&str>) -> Option { + requested_device + .filter(|device| is_cpu_startup_device(Some(device))) + .map(|_| "CPU".to_string()) +} + +fn resolve_startup_backend_device_by_name<'a>( + requested_device: &str, + gpus: &'a [hardware::GpuFacts], +) -> Result<&'a hardware::GpuFacts> { + let matches: Vec<&hardware::GpuFacts> = gpus + .iter() + .filter(|gpu| { + gpu.backend_device.as_deref().is_some_and(|backend_device| { + backend::backend_device_names_match(backend_device, requested_device) + }) + }) + .collect(); + + match matches.as_slice() { + [gpu] => Ok(gpu), + [] => anyhow::bail!( + "requested device '{requested_device}' did not match any detected GPU backend device. Available devices: {}", + display_available_backend_devices(gpus) + ), + _ => anyhow::bail!( + "requested device '{requested_device}' matched multiple detected GPU backend devices. Available devices: {}", + display_available_backend_devices(gpus) + ), + } +} + +fn display_available_backend_devices(gpus: &[hardware::GpuFacts]) -> String { + let names: Vec<&str> = gpus + .iter() + .filter_map(|gpu| gpu.backend_device.as_deref()) + .collect(); + if names.is_empty() { + "none".to_string() + } else { + names.join(", ") + } +} + #[cfg_attr( not(test), expect( diff --git a/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs b/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs index 66db2ac44a..1913ee6b3e 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs @@ -592,40 +592,7 @@ fn test_build_serving_list_keeps_synthetic_local_ref() { assert_eq!(result.len(), 1); } -#[test] -fn test_build_startup_model_specs_prefers_cli_models_over_config() { - let options = runtime_options_for_test(&[ - "mesh-llm", - "--model", - "Qwen3-8B-Q4_K_M", - "--ctx-size", - "4096", - ]); - let config = plugin::MeshConfig { - models: vec![plugin::ModelConfigEntry { - model: "Ignored-Model".into(), - mmproj: Some("/tmp/ignored-mmproj.gguf".into()), - ctx_size: Some(8192), - gpu_id: None, - parallel: None, - cache_type_k: None, - cache_type_v: None, - batch: None, - ubatch: None, - flash_attention: None, - ..Default::default() - }], - ..plugin::MeshConfig::default() - }; - - let specs = build_startup_model_specs(&options, &config).unwrap(); - assert_eq!(specs.len(), 1); - assert_eq!(specs[0].model_ref, PathBuf::from("Qwen3-8B-Q4_K_M")); - assert_eq!(specs[0].mmproj_ref, None); - assert_eq!(specs[0].ctx_size, Some(4096)); - assert_eq!(specs[0].gpu_id, None); - assert!(!specs[0].config_owned); -} +mod cli_device_and_config_matching; #[tokio::test] async fn prepare_runtime_startup_defers_model_resolution_until_after_surfaces() { @@ -696,14 +663,14 @@ fn test_build_startup_model_specs_uses_config_models_when_cli_is_empty() { assert_eq!(specs[0].model_ref, PathBuf::from("Qwen3-8B-Q4_K_M")); assert_eq!(specs[0].ctx_size, Some(4096)); assert_eq!(specs[0].gpu_id, None); - assert!(specs[0].config_owned); + assert!(specs[0].resolve_pinned_gpu); assert_eq!( specs[1].mmproj_ref, Some(PathBuf::from("bartowski/Qwen2.5-VL/mmproj.gguf")) ); assert_eq!(specs[1].ctx_size, Some(4096)); assert_eq!(specs[1].gpu_id, None); - assert!(specs[1].config_owned); + assert!(specs[1].resolve_pinned_gpu); } #[test] @@ -743,7 +710,7 @@ alias = "public-model" specs[1].config_model_id.as_deref(), Some("canonical/override-model") ); - assert!(specs.iter().all(|spec| spec.config_owned)); + assert!(specs.iter().all(|spec| spec.resolve_pinned_gpu)); } #[test] @@ -775,7 +742,7 @@ fn ad_hoc_gguf_alias_preserves_existing_cli_model_overrides() { Some(projector_path.as_path()) ); assert_eq!(specs[0].ctx_size, Some(8192)); - assert!(!specs[0].config_owned); + assert!(!specs[0].resolve_pinned_gpu); } #[test] @@ -808,7 +775,7 @@ parallel = 3 assert_eq!(specs[0].n_ubatch, Some(192)); assert_eq!(specs[0].flash_attention, FlashAttentionType::Enabled); assert!(!specs[0].profile.is_empty()); - assert!(!specs[0].config_owned); + assert!(!specs[0].resolve_pinned_gpu); } #[test] @@ -924,11 +891,24 @@ fn gguf_with_plain_model_name_binds_the_name_to_the_local_file() { "deepseek-v4-flash", ]); - let specs = - build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs"); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + models: vec![plugin::ModelConfigEntry { + model: "deepseek-v4-flash".into(), + gpu_id: Some("pci:0000:65:00.0".into()), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + let specs = build_startup_model_specs(&options, &config).expect("startup specs"); assert_eq!(specs.len(), 1); assert_eq!(specs[0].model_ref, model_path); assert_eq!(specs[0].declared_ref.as_deref(), Some("deepseek-v4-flash")); + assert_eq!(specs[0].gpu_id, None); + assert!(!specs[0].resolve_pinned_gpu); } #[test] @@ -1024,6 +1004,7 @@ async fn local_model_only_rejects_catalog_and_relative_model_refs() { mmproj_ref: None, ctx_size: None, gpu_id: None, + cli_device_override: false, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1031,7 +1012,7 @@ async fn local_model_only_rejects_catalog_and_relative_model_refs() { n_ubatch: None, flash_attention: FlashAttentionType::Auto, profile: "default".into(), - config_owned: false, + resolve_pinned_gpu: false, }]; let error = resolve_local_model_only_startup_models(&specs) @@ -1167,11 +1148,11 @@ fn pinned_gpu_startup_preflight_uses_config_gpu_id() { let specs = build_startup_model_specs(&options, &config).unwrap(); let mut plans = vec![StartupModelPlan { declared_ref: "Qwen3-8B-Q4_K_M".into(), - config_model_id: None, resolved_path: PathBuf::from("/tmp/Qwen3-8B-Q4_K_M.gguf"), mmproj_path: None, ctx_size: Some(8192), gpu_id: specs[0].gpu_id.clone(), + config_model_id: specs[0].config_model_id.clone(), pinned_gpu: None, parallel: None, cache_type_k: None, @@ -1186,8 +1167,7 @@ fn pinned_gpu_startup_preflight_uses_config_gpu_id() { synthetic_gpu(1, Some("pci:0000:b3:00.0"), Some("CUDA1")), ]; - preflight_config_owned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) - .unwrap(); + preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None).unwrap(); assert_eq!(plans[0].gpu_id.as_deref(), Some("pci:0000:65:00.0")); assert_eq!( @@ -1231,7 +1211,8 @@ fn pinned_gpu_startup_preflight_rejects_synthesized_backend_missing_from_probe() mmproj_ref: None, ctx_size: Some(4096), gpu_id: Some("pci:0000:b3:00.0".into()), - config_owned: true, + cli_device_override: false, + resolve_pinned_gpu: true, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1263,7 +1244,7 @@ fn pinned_gpu_startup_preflight_rejects_synthesized_backend_missing_from_probe() available_devices: vec!["Vulkan0".into(), "CPU".into()], }; - let err = preflight_config_owned_startup_models_with_gpus( + let err = preflight_pinned_startup_models_with_gpus( &config, &specs, &mut plans, @@ -1294,7 +1275,8 @@ fn pinned_gpu_startup_preflight_canonicalizes_rocm_hip_alias_from_probe() { mmproj_ref: None, ctx_size: Some(4096), gpu_id: Some("pci:0000:b3:00.0".into()), - config_owned: true, + cli_device_override: false, + resolve_pinned_gpu: true, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1326,7 +1308,7 @@ fn pinned_gpu_startup_preflight_canonicalizes_rocm_hip_alias_from_probe() { available_devices: vec!["HIP1".into(), "CPU".into()], }; - preflight_config_owned_startup_models_with_gpus( + preflight_pinned_startup_models_with_gpus( &config, &specs, &mut plans, @@ -1394,7 +1376,7 @@ fn skippy_telemetry_debug_keeps_debug_level_when_endpoint_is_set() { } #[test] -fn pinned_gpu_startup_preflight_cli_models_bypass_config_gpu_id() { +fn pinned_gpu_startup_preflight_unmatched_cli_models_bypass_config_gpu_id() { let options = runtime_options_for_test(&["mesh-llm", "--model", "Qwen3-8B-Q4_K_M"]); let config = plugin::MeshConfig { gpu: plugin::GpuConfig { @@ -1419,11 +1401,11 @@ fn pinned_gpu_startup_preflight_cli_models_bypass_config_gpu_id() { let specs = build_startup_model_specs(&options, &config).unwrap(); let mut plans = vec![StartupModelPlan { declared_ref: "Qwen3-8B-Q4_K_M".into(), - config_model_id: None, resolved_path: PathBuf::from("/tmp/Qwen3-8B-Q4_K_M.gguf"), mmproj_path: None, ctx_size: None, gpu_id: specs[0].gpu_id.clone(), + config_model_id: specs[0].config_model_id.clone(), pinned_gpu: None, parallel: None, cache_type_k: None, @@ -1435,11 +1417,10 @@ fn pinned_gpu_startup_preflight_cli_models_bypass_config_gpu_id() { }]; let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; - preflight_config_owned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) - .unwrap(); + preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None).unwrap(); assert_eq!(specs[0].gpu_id, None); - assert!(!specs[0].config_owned); + assert!(!specs[0].resolve_pinned_gpu); assert_eq!(plans[0].gpu_id, None); assert_eq!(plans[0].pinned_gpu, None); } @@ -1460,7 +1441,8 @@ fn pinned_gpu_startup_preflight_missing_gpu_id_fails_closed() { mmproj_ref: None, ctx_size: None, gpu_id: None, - config_owned: true, + cli_device_override: false, + resolve_pinned_gpu: true, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1487,9 +1469,8 @@ fn pinned_gpu_startup_preflight_missing_gpu_id_fails_closed() { }]; let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; - let err = - preflight_config_owned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) - .unwrap_err(); + let err = preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .unwrap_err(); let message = format!("{err:#}"); assert!(message.contains("failed pinned GPU preflight")); @@ -1512,7 +1493,8 @@ fn pinned_gpu_startup_preflight_stores_resolved_pinned_target_in_plan() { mmproj_ref: None, ctx_size: Some(4096), gpu_id: Some("uuid:GPU-123".into()), - config_owned: true, + cli_device_override: false, + resolve_pinned_gpu: true, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1540,8 +1522,7 @@ fn pinned_gpu_startup_preflight_stores_resolved_pinned_target_in_plan() { let mut gpus = vec![synthetic_gpu(3, Some("uuid:GPU-123"), Some("CUDA3"))]; gpus[0].reserved_bytes = Some(500_000_000); - preflight_config_owned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) - .unwrap(); + preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None).unwrap(); let pinned_gpu = plans[0].pinned_gpu.as_ref().unwrap(); assert_eq!(pinned_gpu.index, 3); @@ -1567,7 +1548,8 @@ fn pinned_gpu_startup_preflight_rejects_resolved_gpu_without_backend_device() { mmproj_ref: None, ctx_size: Some(4096), gpu_id: Some("uuid:GPU-123".into()), - config_owned: true, + cli_device_override: false, + resolve_pinned_gpu: true, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1594,9 +1576,8 @@ fn pinned_gpu_startup_preflight_rejects_resolved_gpu_without_backend_device() { }]; let gpus = vec![synthetic_gpu(3, Some("uuid:GPU-123"), None)]; - let err = - preflight_config_owned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) - .unwrap_err(); + let err = preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .unwrap_err(); let message = format!("{err:#}"); assert!(message.contains("failed pinned GPU preflight")); @@ -1619,7 +1600,8 @@ fn pinned_gpu_startup_preflight_unresolvable_gpu_id_fails_closed() { mmproj_ref: None, ctx_size: None, gpu_id: Some("pci:0000:b3:00.0".into()), - config_owned: true, + cli_device_override: false, + resolve_pinned_gpu: true, parallel: None, cache_type_k: None, cache_type_v: None, @@ -1646,9 +1628,8 @@ fn pinned_gpu_startup_preflight_unresolvable_gpu_id_fails_closed() { }]; let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; - let err = - preflight_config_owned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) - .unwrap_err(); + let err = preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .unwrap_err(); let message = format!("{err:#}"); assert!(message.contains("failed pinned GPU preflight")); @@ -1677,7 +1658,8 @@ fn test_should_not_show_serve_config_help_when_models_are_present() { mmproj_ref: None, ctx_size: None, gpu_id: None, - config_owned: false, + cli_device_override: false, + resolve_pinned_gpu: false, parallel: None, cache_type_k: None, cache_type_v: None, diff --git a/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models/cli_device_and_config_matching.rs b/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models/cli_device_and_config_matching.rs new file mode 100644 index 0000000000..76aaf7f733 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models/cli_device_and_config_matching.rs @@ -0,0 +1,550 @@ +use super::*; + +#[test] +fn test_build_startup_model_specs_prefers_cli_models_over_config() { + let options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--ctx-size", + "4096", + ]); + let config = plugin::MeshConfig { + models: vec![plugin::ModelConfigEntry { + model: "Ignored-Model".into(), + mmproj: Some("/tmp/ignored-mmproj.gguf".into()), + ctx_size: Some(8192), + gpu_id: None, + parallel: None, + cache_type_k: None, + cache_type_v: None, + batch: None, + ubatch: None, + flash_attention: None, + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].model_ref, PathBuf::from("Qwen3-8B-Q4_K_M")); + assert_eq!(specs[0].mmproj_ref, None); + assert_eq!(specs[0].ctx_size, Some(4096)); + assert_eq!(specs[0].gpu_id, None); + assert!(!specs[0].resolve_pinned_gpu); +} +#[test] +fn cli_model_exact_config_ref_resolves_pinned_backend_and_keeps_cli_overrides() { + let mut options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--ctx-size", + "4096", + ]); + options.mmproj = Some(PathBuf::from("/tmp/cli-mmproj.gguf")); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + models: vec![plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + mmproj: Some("/tmp/config-mmproj.gguf".into()), + ctx_size: Some(8192), + gpu_id: None, + parallel: Some(8), + hardware: Some(plugin::HardwareConfig { + device: Some("pci:0000:65:00.0".into()), + model_path: Some("/configured/model.gguf".into()), + ..Default::default() + }), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].model_ref, PathBuf::from("Qwen3-8B-Q4_K_M")); + assert_eq!(specs[0].gpu_id.as_deref(), Some("pci:0000:65:00.0")); + assert!(specs[0].resolve_pinned_gpu); + assert_eq!(specs[0].config_model_id, None); + assert_eq!(specs[0].ctx_size, Some(4096)); + assert_eq!( + specs[0].mmproj_ref, + Some(PathBuf::from("/tmp/cli-mmproj.gguf")) + ); + assert_eq!(specs[0].parallel, None); + + let mut plans = vec![StartupModelPlan { + declared_ref: "Qwen3-8B-Q4_K_M".into(), + resolved_path: PathBuf::from("/tmp/Qwen3-8B-Q4_K_M.gguf"), + mmproj_path: specs[0].mmproj_ref.clone(), + ctx_size: specs[0].ctx_size, + gpu_id: specs[0].gpu_id.clone(), + config_model_id: specs[0].config_model_id.clone(), + pinned_gpu: None, + parallel: specs[0].parallel, + cache_type_k: None, + cache_type_v: None, + n_batch: None, + n_ubatch: None, + flash_attention: FlashAttentionType::Auto, + profile: String::new(), + }]; + let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; + + preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .expect("exact CLI config ref should resolve its pinned GPU"); + assert_eq!( + plans[0].pinned_gpu.as_ref().unwrap().backend_device, + "CUDA0" + ); +} + +#[test] +fn cli_device_overrides_a_persisted_model_pin() { + let options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--device", + "CUDA1", + ]); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + models: vec![plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + gpu_id: Some("pci:0000:65:00.0".into()), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs[0].gpu_id.as_deref(), Some("CUDA1")); + assert!(specs[0].cli_device_override); + + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("Qwen3-8B-Q4_K_M") + }]; + let gpus = vec![ + synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0")), + synthetic_gpu(1, Some("pci:0000:b3:00.0"), Some("CUDA1")), + ]; + preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None).unwrap(); + + assert_eq!( + plans[0] + .pinned_gpu + .as_ref() + .map(|gpu| gpu.backend_device.as_str()), + Some("CUDA1") + ); +} + +#[test] +fn cli_device_resolves_under_auto_assignment() { + let options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--device", + "CUDA0", + ]); + let config = plugin::MeshConfig::default(); + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert!(specs[0].cli_device_override); + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("Qwen3-8B-Q4_K_M") + }]; + let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; + + preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None).unwrap(); + + assert_eq!( + plans[0] + .pinned_gpu + .as_ref() + .map(|gpu| gpu.backend_device.as_str()), + Some("CUDA0") + ); +} + +#[test] +fn cli_device_auto_falls_back_to_the_persisted_pin() { + let options = + runtime_options_for_test(&["mesh-llm", "--model", "Qwen3-8B-Q4_K_M", "--device", "Auto"]); + let config = plugin::MeshConfig { + models: vec![plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + gpu_id: Some("pci:0000:65:00.0".into()), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + + assert_eq!(specs[0].gpu_id.as_deref(), Some("pci:0000:65:00.0")); + assert!(!specs[0].cli_device_override); +} + +#[test] +fn persisted_gpu_id_rejects_backend_device_name() { + let options = runtime_options_for_test(&["mesh-llm"]); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + models: vec![plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + gpu_id: Some("CUDA0".into()), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("Qwen3-8B-Q4_K_M") + }]; + let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; + + let error = preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .expect_err("persisted gpu_id must remain a stable GPU ID"); + + assert!(format!("{error:#}").contains("not pinnable")); +} + +#[test] +fn persisted_default_device_rejects_backend_device_name() { + let options = runtime_options_for_test(&["mesh-llm", "--model", "ad-hoc-model"]); + let config = plugin::MeshConfig { + defaults: Some(plugin::ModelConfigDefaults { + hardware: Some(plugin::HardwareConfig { + device: Some("CUDA0".into()), + ..Default::default() + }), + ..Default::default() + }), + ..plugin::MeshConfig::default() + }; + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("ad-hoc-model") + }]; + let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; + + let error = preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .expect_err("persisted default device must remain a stable GPU ID"); + + assert!(format!("{error:#}").contains("not pinnable")); +} + +#[test] +fn unresolved_cli_device_names_the_available_devices() { + let options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--device", + "CUDA9", + ]); + let config = plugin::MeshConfig::default(); + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("Qwen3-8B-Q4_K_M") + }]; + let gpus = vec![synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))]; + + let error = preflight_pinned_startup_models_with_gpus(&config, &specs, &mut plans, &gpus, None) + .unwrap_err(); + let message = format!("{error:#}"); + assert!(message.contains("did not match any detected GPU backend device")); + assert!(message.contains("Available devices: CUDA0")); +} + +#[test] +fn cli_device_reaches_the_outer_preflight_under_auto_assignment() { + let options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--device", + "pci:0000:ff:ff.7", + ]); + let config = plugin::MeshConfig::default(); + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("Qwen3-8B-Q4_K_M") + }]; + + let error = preflight_pinned_startup_models(&config, &specs, &mut plans, None, None) + .expect_err("an impossible CLI device must fail before native startup"); + + assert!(format!("{error:#}").contains("failed pinned GPU preflight")); +} + +#[test] +fn cli_cpu_device_bypasses_gpu_preflight() { + let options = + runtime_options_for_test(&["mesh-llm", "--model", "Qwen3-8B-Q4_K_M", "--device", "CPU"]); + let config = plugin::MeshConfig::default(); + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![StartupModelPlan { + gpu_id: specs[0].gpu_id.clone(), + ..startup_model_plan("Qwen3-8B-Q4_K_M") + }]; + + preflight_pinned_startup_models(&config, &specs, &mut plans, None, None).unwrap(); + + assert_eq!(plans[0].gpu_id.as_deref(), Some("CPU")); + assert_eq!(plans[0].pinned_gpu, None); + assert_eq!( + startup_device_override(plans[0].gpu_id.as_deref()).as_deref(), + Some("CPU") + ); +} + +#[test] +fn auto_assignment_without_a_device_stays_inert() { + let options = runtime_options_for_test(&["mesh-llm", "--model", "Qwen3-8B-Q4_K_M"]); + let config = plugin::MeshConfig::default(); + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![startup_model_plan("Qwen3-8B-Q4_K_M")]; + + preflight_pinned_startup_models(&config, &specs, &mut plans, None, None).unwrap(); + + assert_eq!(plans[0].pinned_gpu, None); +} + +#[test] +fn cli_model_exact_config_ref_without_gpu_fails_before_launch() { + let options = runtime_options_for_test(&["mesh-llm", "--model", "configured/model"]); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + models: vec![plugin::ModelConfigEntry { + model: "configured/model".into(), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + let specs = build_startup_model_specs(&options, &config).unwrap(); + let mut plans = vec![startup_model_plan("configured/model")]; + + let error = preflight_pinned_startup_models_with_gpus( + &config, + &specs, + &mut plans, + &[synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))], + None, + ) + .expect_err("a selected configured model without a GPU must fail preflight"); + let message = format!("{error:#}"); + assert!(message.contains("startup model 'configured/model'")); + assert!(message.contains("missing configured gpu_id")); +} + +#[test] +fn cli_model_matching_duplicate_config_refs_fails_as_ambiguous() { + let options = runtime_options_for_test(&["mesh-llm", "--model", "Qwen3-8B-Q4_K_M"]); + let config = plugin::MeshConfig { + models: vec![ + plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + gpu_id: Some("pci:0000:65:00.0".into()), + ctx_size: Some(4096), + ..Default::default() + }, + plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + gpu_id: Some("pci:0000:b3:00.0".into()), + ctx_size: Some(8192), + ..Default::default() + }, + ], + ..plugin::MeshConfig::default() + }; + + let error = build_startup_model_specs(&options, &config) + .expect_err("a CLI ref cannot select between duplicate configured profiles"); + let message = format!("{error:#}"); + assert!(message.contains("matches multiple configured model entries")); + assert!(message.contains("Qwen3-8B-Q4_K_M")); +} + +#[test] +fn cli_gguf_does_not_match_configured_model_path_for_pinned_gpu() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let model_path = temp_dir.path().join("configured.gguf"); + std::fs::write(&model_path, b"gguf").expect("write model"); + let options = runtime_options_for_test(&[ + "mesh-llm", + "--gguf", + model_path.to_str().expect("model path"), + ]); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + models: vec![plugin::ModelConfigEntry { + model: "configured/model-ref".into(), + gpu_id: Some("pci:0000:65:00.0".into()), + hardware: Some(plugin::HardwareConfig { + model_path: Some(model_path.display().to_string()), + ..Default::default() + }), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].model_ref, model_path); + assert_eq!(specs[0].gpu_id, None); + assert!(!specs[0].resolve_pinned_gpu); + assert_eq!(specs[0].config_model_id, None); +} + +#[test] +fn cli_gguf_inherits_global_pinned_default_without_model_ownership() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let model_path = temp_dir.path().join("selected.gguf"); + std::fs::write(&model_path, b"gguf").expect("write model"); + let options = runtime_options_for_test(&[ + "mesh-llm", + "--gguf", + model_path.to_str().expect("model path"), + ]); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + defaults: Some(plugin::ModelConfigDefaults { + hardware: Some(plugin::HardwareConfig { + device: Some("pci:0000:65:00.0".into()), + ..Default::default() + }), + ..Default::default() + }), + models: vec![plugin::ModelConfigEntry { + model: "configured/model-ref".into(), + hardware: Some(plugin::HardwareConfig { + model_path: Some(model_path.display().to_string()), + device: Some("pci:0000:b3:00.0".into()), + ..Default::default() + }), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].model_ref, model_path); + assert_eq!(specs[0].gpu_id.as_deref(), Some("pci:0000:65:00.0")); + assert!(!specs[0].resolve_pinned_gpu); + assert_eq!(specs[0].config_model_id, None); + + let mut plans = vec![startup_model_plan(model_path.to_str().expect("model path"))]; + plans[0].gpu_id = specs[0].gpu_id.clone(); + preflight_pinned_startup_models_with_gpus( + &config, + &specs, + &mut plans, + &[synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))], + None, + ) + .expect("global pinned default should resolve for an explicit gguf model"); + assert_eq!( + plans[0].pinned_gpu.as_ref().unwrap().backend_device, + "CUDA0" + ); +} + +#[test] +fn cli_model_matching_is_independent_for_multiple_models() { + let options = runtime_options_for_test(&[ + "mesh-llm", + "--model", + "Qwen3-8B-Q4_K_M", + "--model", + "ad-hoc-model", + ]); + let config = plugin::MeshConfig { + models: vec![plugin::ModelConfigEntry { + model: "Qwen3-8B-Q4_K_M".into(), + gpu_id: Some("pci:0000:65:00.0".into()), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].gpu_id.as_deref(), Some("pci:0000:65:00.0")); + assert!(specs[0].resolve_pinned_gpu); + assert_eq!(specs[1].gpu_id, None); + assert!(!specs[1].resolve_pinned_gpu); +} + +#[test] +fn cli_unmatched_model_uses_global_pinned_default_without_model_ownership() { + let options = runtime_options_for_test(&["mesh-llm", "--model", "ad-hoc-model"]); + let config = plugin::MeshConfig { + gpu: plugin::GpuConfig { + assignment: plugin::GpuAssignment::Pinned, + parallel: None, + }, + defaults: Some(plugin::ModelConfigDefaults { + hardware: Some(plugin::HardwareConfig { + device: Some("pci:0000:65:00.0".into()), + ..Default::default() + }), + ..Default::default() + }), + models: vec![plugin::ModelConfigEntry { + model: "configured/model".into(), + gpu_id: Some("pci:0000:b3:00.0".into()), + ..Default::default() + }], + ..plugin::MeshConfig::default() + }; + + let specs = build_startup_model_specs(&options, &config).unwrap(); + assert_eq!(specs[0].gpu_id.as_deref(), Some("pci:0000:65:00.0")); + assert!(!specs[0].resolve_pinned_gpu); + assert_eq!(specs[0].config_model_id, None); + + let mut plans = vec![startup_model_plan("ad-hoc-model")]; + plans[0].gpu_id = specs[0].gpu_id.clone(); + preflight_pinned_startup_models_with_gpus( + &config, + &specs, + &mut plans, + &[synthetic_gpu(0, Some("pci:0000:65:00.0"), Some("CUDA0"))], + None, + ) + .expect("global pinned default should resolve for an ad-hoc CLI model"); + assert_eq!( + plans[0].pinned_gpu.as_ref().unwrap().backend_device, + "CUDA0" + ); +} diff --git a/docs/USAGE.md b/docs/USAGE.md index f273a899c5..f9a824f90a 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -839,9 +839,17 @@ Config precedence: frontend boundary. Explicit request values win, and those defaults never become `StageConfig`, runtime load structs, protobuf payloads, or lower-layer runtime settings. -- Explicit `--model` or `--gguf` ignores configured `[[models]]`. +- Explicit `--model` or `--gguf` ignores configured `[[models]]` for model + selection and tuning. An exact, unique `--model` ref may still inherit its + configured pinned GPU selector. Unmatched `--model` refs and `--gguf` paths + carry no configured model identity but may inherit only + `defaults.hardware.device`. - Explicit `--ctx-size` overrides configured `ctx_size` for the selected startup models. +- Explicit `--device` overrides the inherited device selector. Stable GPU IDs + and backend names such as `CUDA0` resolve before native startup, including + under automatic assignment. `--device CPU` selects CPU without requiring a + GPU inventory, while `--device auto` keeps the inherited selector. - Explicit `--mesh-guardrails ` seeds the server-side mesh guardrail mode for hosted Skippy startup models and later runtime-loaded models. diff --git a/docs/design/DESIGN.md b/docs/design/DESIGN.md index 984d134e7c..d5d5ec8d67 100644 --- a/docs/design/DESIGN.md +++ b/docs/design/DESIGN.md @@ -185,8 +185,13 @@ Phase 2 keeps this config intentionally local-node only. There is no authored me CLI precedence is by concern: -- explicit `--model` or `--gguf` ignores configured `[[models]]` +- explicit `--model` or `--gguf` ignores configured `[[models]]` for model + selection and tuning; an exact, unique `--model` ref may still inherit its + configured pinned GPU selector - explicit `--ctx-size` overrides configured `ctx_size` +- explicit `--device` wins over configured device selectors; stable IDs and + backend names resolve before startup, `CPU` bypasses GPU preflight, and + `auto` keeps the inherited selector - plugin config continues to load from the same file Pinned GPU startup is also local-node only: @@ -194,7 +199,12 @@ Pinned GPU startup is also local-node only: - `[gpu].assignment = "pinned"` means each configured `[[models]]` entry must carry its own `gpu_id` - valid IDs come from the local `mesh-llm gpus` / `mesh-llm gpus --json` inventory surface - pin resolution is host-local and fail-closed: missing, ambiguous, unsupported, or stale IDs abort startup and config push for that node instead of silently falling back to auto placement -- explicit CLI `--model` / `--gguf` still bypass configured `[[models]]`, so they do not inherit config-owned pinned IDs +- explicit CLI `--model` / `--gguf` keeps the selected artifact, context, and + projector choices. An exact, unique `--model` ref carries only its effective + pinned GPU selector. Unmatched refs and `--gguf` paths carry no configured + model identity but may inherit only `defaults.hardware.device`; duplicate + configured refs are rejected because the CLI has no profile selector +- an explicit `--device` is resolved even when `[gpu].assignment = "auto"` Bare `mesh-llm serve` is the config-owned path. If `[[models]]` is empty, it warns, prints help, and exits cleanly. Background services use that path directly. diff --git a/docs/design/TESTING.md b/docs/design/TESTING.md index 64447ea8ba..279886cc41 100644 --- a/docs/design/TESTING.md +++ b/docs/design/TESTING.md @@ -67,8 +67,13 @@ mesh-llm serve - Both configured startup models should be considered for launch - If `[[models]]` is empty, `mesh-llm serve` should print a `⚠️` warning, show help, and exit cleanly -- Explicit `--model` or `--gguf` should ignore configured `[[models]]` +- Explicit `--model` or `--gguf` should ignore configured `[[models]]` for + model selection and tuning, except that an exact, unique `--model` ref may + inherit its configured pinned GPU selector - Explicit `--ctx-size` should override configured `ctx_size` +- Explicit `--device` should override persisted device selectors under pinned + or automatic assignment. Backend names resolve to a detected device, `CPU` + bypasses GPU-only preflight, and `auto` retains the inherited selector. - `mesh-llm benchmark tune` is the measured local model-serving tuning companion for these startup configs. It only accepts already-downloaded targets, rejects remote-only or not-downloaded refs without fetching them, and runs isolated throughput trials. For speculative decoding changes, run a small sweep that includes the disabled baseline plus `mtp`, `mtp-ngram`, or draft candidates as applicable, then inspect trial logs/telemetry for native MTP or draft acceptance statistics in addition to decode tok/s. ### 0b. Pinned startup smoke @@ -101,8 +106,16 @@ mesh-llm serve - Startup should succeed only when `gpu_id` matches a valid local pinnable stable ID from `mesh-llm gpus` - If the pinned ID is missing, ambiguous, unsupported, or stale, startup should fail closed before local launch -- Explicit `mesh-llm serve --model ...` should still bypass configured `[[models]]` and therefore bypass config-owned pinned IDs +- Explicit `mesh-llm serve --model ...` keeps the CLI model path, context, and + projector choices. When the ref exactly matches one configured model, only + that model's effective pinned GPU selector is carried forward and resolved + from its stable ID to the backend device name. Unmatched refs and all + `--gguf` paths carry no configured model identity but may inherit only + `defaults.hardware.device`; duplicate configured refs are rejected as + ambiguous because the CLI has no profile selector. - Do not use GPU indexes, `index:*`, or backend-device names like `CUDA0` / `HIP0` / `MTL0` as `gpu_id` +- Backend-device names are accepted only through the explicit CLI `--device` + override. Persisted `gpu_id` values remain stable IDs. ### 0c. Requirement-aware mesh smoke diff --git a/tools/xtask/data/console_print_allowlist.json b/tools/xtask/data/console_print_allowlist.json index 7e93977808..d085f04fd5 100644 --- a/tools/xtask/data/console_print_allowlist.json +++ b/tools/xtask/data/console_print_allowlist.json @@ -1885,19 +1885,19 @@ ], "crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs": [ { - "line": 1870, + "line": 1871, "macro_name": "eprintln!" }, { - "line": 1875, + "line": 1876, "macro_name": "eprintln!" }, { - "line": 1888, + "line": 1889, "macro_name": "eprintln!" }, { - "line": 1910, + "line": 1911, "macro_name": "eprintln!" } ],