Add unrecoverable sandbox state

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

Author: ludfjig

CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## [Prerelease] - Unreleased
 
 ### Added
+* Add `MultiUseSandbox::status()`, which returns `SandboxStatus` for inspecting sandbox lifecycle state.
 
 ### Changed
 * **Breaking:** Guest MSR state is now saved and restored across snapshots.
@@ -13,10 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
   resets to a clean default. On KVM the guest may only read or write declared
   MSRs, on MSHV and WHP this is not enforced. by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/991
 * **Breaking:** Filesystem paths are now represented using `PathBuf`. `GuestBinary::FilePath` now stores a `PathBuf` instead of a `String`, and `MultiUseSandbox::generate_crashdump_to_dir` accepts `Into<PathBuf>` instead of `Into<String>`. Callers passing a `String` to `GuestBinary::FilePath` must convert it using `.into()`.
+* Deprecate `MultiUseSandbox::poisoned` in favor of `MultiUseSandbox::status().is_poisoned()`.
 
 ### Removed
 
 ### Fixed
+* Mark a sandbox unrecoverable when snapshot restore fails while updating its VM mappings.
 * Fix symbol resolution in guest core dumps for sandboxes created from snapshots by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/1618
 * Reject malformed OCI snapshot metadata and non-regular artifact files during load.
 * Reset XCR0 during x86 snapshot restore.

src/hyperlight_host/src/error.rs

@@ -224,6 +224,10 @@ pub enum HyperlightError {
     #[error("The sandbox was poisoned")]
     PoisonedSandbox,
 
+    /// The sandbox cannot safely perform further operations.
+    #[error("The sandbox is unrecoverable and must be discarded")]
+    UnrecoverableSandbox,
+
     /// Raw pointer is less than base address
     #[error("Raw pointer ({0:?}) was less than the base address ({1})")]
     RawPointerLessThanBaseAddress(RawPtr, u64),
@@ -408,6 +412,7 @@ impl HyperlightError {
             | HyperlightError::UnexpectedNoOfArguments(_, _)
             | HyperlightError::UnexpectedParameterValueType(_, _)
             | HyperlightError::UnexpectedReturnValueType(_, _)
+            | HyperlightError::UnrecoverableSandbox
             | HyperlightError::UTF8StringConversionFailure(_)
             | HyperlightError::VectorCapacityIncorrect(_, _, _) => false,
 

src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs

@@ -19,6 +19,8 @@ mod x86_64;
 
 #[cfg(target_arch = "aarch64")]
 mod aarch64;
+#[cfg(all(test, not(gdb), any(kvm, mshv3, target_os = "windows")))]
+pub(crate) mod test_support;
 #[cfg(gdb)]
 use std::collections::HashMap;
 use std::str::FromStr;
@@ -532,11 +534,12 @@ impl HyperlightVm {
         let guest_base = crate::mem::layout::SandboxMemoryLayout::BASE_ADDRESS as u64;
         let rgn = snapshot.mapping_at(guest_base, MemoryRegionType::Snapshot);
 
-        if let Some(old_snapshot) = self.snapshot_memory.replace(snapshot) {
+        if let Some(old_snapshot) = self.snapshot_memory.as_ref() {
             let old_rgn = old_snapshot.mapping_at(guest_base, MemoryRegionType::Snapshot);
             self.vm.unmap_memory((self.snapshot_slot, &old_rgn))?;
         }
         unsafe { self.vm.map_memory((self.snapshot_slot, &rgn))? };
+        self.snapshot_memory = Some(snapshot);
 
         Ok(())
     }
@@ -549,12 +552,13 @@ impl HyperlightVm {
         let guest_base = hyperlight_common::layout::scratch_base_gpa(scratch.mem_size());
         let rgn = scratch.mapping_at(guest_base, MemoryRegionType::Scratch);
 
-        if let Some(old_scratch) = self.scratch_memory.replace(scratch) {
+        if let Some(old_scratch) = self.scratch_memory.as_ref() {
             let old_base = hyperlight_common::layout::scratch_base_gpa(old_scratch.mem_size());
             let old_rgn = old_scratch.mapping_at(old_base, MemoryRegionType::Scratch);
             self.vm.unmap_memory((self.scratch_slot, &old_rgn))?;
         }
         unsafe { self.vm.map_memory((self.scratch_slot, &rgn))? };
+        self.scratch_memory = Some(scratch);
 
         Ok(())
     }

src/hyperlight_host/src/hypervisor/hyperlight_vm/test_support.rs

@@ -0,0 +1,292 @@
+/*
+Copyright 2025 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use std::collections::VecDeque;
+
+use super::*;
+#[cfg(target_arch = "x86_64")]
+use crate::hypervisor::regs::MsrEntry;
+use crate::hypervisor::regs::{
+    CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters,
+};
+use crate::hypervisor::virtual_machine::{CreateVmError, HypervisorError};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum VmOperation {
+    Map(MemoryRegionType),
+    Unmap(MemoryRegionType),
+    #[cfg(target_arch = "x86_64")]
+    SetRegs,
+    #[cfg(target_arch = "x86_64")]
+    SetDebugRegs,
+    #[cfg(target_arch = "x86_64")]
+    ResetXsave,
+    #[cfg(target_arch = "x86_64")]
+    SetSregs,
+    #[cfg(target_arch = "x86_64")]
+    SetMsrs,
+    #[cfg(target_arch = "aarch64")]
+    ResetVcpu,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct VmFaultPlan {
+    operations: Arc<Mutex<VecDeque<VmOperation>>>,
+}
+
+impl VmFaultPlan {
+    fn new(operations: impl IntoIterator<Item = VmOperation>) -> Self {
+        Self {
+            operations: Arc::new(Mutex::new(operations.into_iter().collect())),
+        }
+    }
+
+    pub(crate) fn is_consumed(&self) -> bool {
+        self.operations.lock().unwrap().is_empty()
+    }
+
+    fn should_fail(&self, operation: VmOperation) -> bool {
+        let mut operations = self.operations.lock().unwrap();
+        if operations.front() == Some(&operation) {
+            operations.pop_front();
+            true
+        } else {
+            false
+        }
+    }
+}
+
+#[derive(Debug)]
+struct FaultInjectingVirtualMachine {
+    inner: Option<Box<dyn VirtualMachine>>,
+    fault_plan: VmFaultPlan,
+}
+
+impl FaultInjectingVirtualMachine {
+    fn new(
+        inner: Box<dyn VirtualMachine>,
+        operations: impl IntoIterator<Item = VmOperation>,
+    ) -> (Self, VmFaultPlan) {
+        let fault_plan = VmFaultPlan::new(operations);
+        (
+            Self {
+                inner: Some(inner),
+                fault_plan: fault_plan.clone(),
+            },
+            fault_plan,
+        )
+    }
+
+    fn placeholder() -> Self {
+        Self {
+            inner: None,
+            fault_plan: VmFaultPlan::new([]),
+        }
+    }
+
+    fn inner(&self) -> &dyn VirtualMachine {
+        self.inner.as_deref().expect("placeholder VM was used")
+    }
+
+    fn inner_mut(&mut self) -> &mut dyn VirtualMachine {
+        self.inner.as_deref_mut().expect("placeholder VM was used")
+    }
+
+    fn should_fail(&self, operation: VmOperation) -> bool {
+        self.fault_plan.should_fail(operation)
+    }
+
+    fn injected_error() -> HypervisorError {
+        #[cfg(kvm)]
+        let error = kvm_ioctls::Error::new(libc::EIO);
+        #[cfg(all(not(kvm), mshv3))]
+        let error = mshv_ioctls::MshvError::from(libc::EIO);
+        #[cfg(target_os = "windows")]
+        let error = windows_result::Error::from_hresult(windows_result::HRESULT::from_win32(5));
+        error.into()
+    }
+}
+
+impl VirtualMachine for FaultInjectingVirtualMachine {
+    unsafe fn map_memory(
+        &mut self,
+        region: (u32, &MemoryRegion),
+    ) -> std::result::Result<(), MapMemoryError> {
+        if self.should_fail(VmOperation::Map(region.1.region_type)) {
+            return Err(MapMemoryError::Hypervisor(Self::injected_error()));
+        }
+        // SAFETY: The decorator forwards the caller's preconditions unchanged.
+        unsafe { self.inner_mut().map_memory(region) }
+    }
+
+    fn unmap_memory(
+        &mut self,
+        region: (u32, &MemoryRegion),
+    ) -> std::result::Result<(), UnmapMemoryError> {
+        if self.should_fail(VmOperation::Unmap(region.1.region_type)) {
+            return Err(UnmapMemoryError::Hypervisor(Self::injected_error()));
+        }
+        self.inner_mut().unmap_memory(region)
+    }
+
+    fn run_vcpu(
+        &mut self,
+        #[cfg(feature = "trace_guest")] tc: &mut crate::sandbox::trace::TraceContext,
+    ) -> std::result::Result<VmExit, RunVcpuError> {
+        self.inner_mut().run_vcpu(
+            #[cfg(feature = "trace_guest")]
+            tc,
+        )
+    }
+
+    fn regs(&self) -> std::result::Result<CommonRegisters, RegisterError> {
+        self.inner().regs()
+    }
+
+    fn set_regs(&self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> {
+        #[cfg(target_arch = "x86_64")]
+        if self.should_fail(VmOperation::SetRegs) {
+            return Err(RegisterError::SetRegs(Self::injected_error()));
+        }
+        self.inner().set_regs(regs)
+    }
+
+    fn fpu(&self) -> std::result::Result<CommonFpu, RegisterError> {
+        self.inner().fpu()
+    }
+
+    fn set_fpu(&self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> {
+        self.inner().set_fpu(fpu)
+    }
+
+    fn sregs(&self) -> std::result::Result<CommonSpecialRegisters, RegisterError> {
+        self.inner().sregs()
+    }
+
+    fn set_sregs(&self, sregs: &CommonSpecialRegisters) -> std::result::Result<(), RegisterError> {
+        #[cfg(target_arch = "x86_64")]
+        if self.should_fail(VmOperation::SetSregs) {
+            return Err(RegisterError::SetSregs(Self::injected_error()));
+        }
+        self.inner().set_sregs(sregs)
+    }
+
+    fn debug_regs(&self) -> std::result::Result<CommonDebugRegs, RegisterError> {
+        self.inner().debug_regs()
+    }
+
+    fn set_debug_regs(&self, drs: &CommonDebugRegs) -> std::result::Result<(), RegisterError> {
+        #[cfg(target_arch = "x86_64")]
+        if self.should_fail(VmOperation::SetDebugRegs) {
+            return Err(RegisterError::SetDebugRegs(Self::injected_error()));
+        }
+        self.inner().set_debug_regs(drs)
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    fn msrs(&self, indices: &[u32]) -> std::result::Result<Vec<MsrEntry>, RegisterError> {
+        self.inner().msrs(indices)
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> {
+        if self.should_fail(VmOperation::SetMsrs) {
+            return Err(RegisterError::SetMsrs(Self::injected_error()));
+        }
+        self.inner().set_msrs(msrs)
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    fn msr_reset_indices(
+        &self,
+        guest_msrs: &[u32],
+    ) -> std::result::Result<Vec<u32>, CreateVmError> {
+        self.inner().msr_reset_indices(guest_msrs)
+    }
+
+    #[cfg(not(target_arch = "aarch64"))]
+    fn xsave(&self) -> std::result::Result<Vec<u8>, RegisterError> {
+        self.inner().xsave()
+    }
+
+    #[cfg(not(target_arch = "aarch64"))]
+    fn reset_xsave(&self) -> std::result::Result<(), RegisterError> {
+        #[cfg(target_arch = "x86_64")]
+        if self.should_fail(VmOperation::ResetXsave) {
+            return Err(RegisterError::SetXsave(Self::injected_error()));
+        }
+        self.inner().reset_xsave()
+    }
+
+    #[cfg(not(target_arch = "aarch64"))]
+    fn set_xsave(&self, xsave: &[u32]) -> std::result::Result<(), RegisterError> {
+        self.inner().set_xsave(xsave)
+    }
+
+    #[cfg(all(test, target_arch = "x86_64"))]
+    fn xcr0(&self) -> std::result::Result<u64, RegisterError> {
+        self.inner().xcr0()
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    fn set_xcr0(&self, value: u64) -> std::result::Result<(), RegisterError> {
+        self.inner().set_xcr0(value)
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    fn can_reset_vcpu(&self) -> bool {
+        self.inner().can_reset_vcpu()
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    fn reset_vcpu(&mut self) -> std::result::Result<(), ResetVcpuError> {
+        if self.should_fail(VmOperation::ResetVcpu) {
+            return Err(ResetVcpuError::Hypervisor(Self::injected_error()));
+        }
+        self.inner_mut().reset_vcpu()
+    }
+
+    #[cfg(target_os = "windows")]
+    fn partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE {
+        self.inner().partition_handle()
+    }
+}
+
+impl HyperlightVm {
+    pub(crate) fn inject_vm_faults(
+        &mut self,
+        operations: impl IntoIterator<Item = VmOperation>,
+    ) -> VmFaultPlan {
+        let placeholder = Box::new(FaultInjectingVirtualMachine::placeholder());
+        let inner = std::mem::replace(&mut self.vm, placeholder);
+        let (vm, fault_plan) = FaultInjectingVirtualMachine::new(inner, operations);
+        self.vm = Box::new(vm);
+        fault_plan
+    }
+
+    #[allow(clippy::type_complexity, reason = "test-only mapping state")]
+    pub(crate) fn base_mapping_state(&self) -> (Option<(usize, usize)>, Option<(usize, usize)>) {
+        let snapshot = self
+            .snapshot_memory
+            .as_ref()
+            .map(|memory| (memory.base_addr(), memory.mem_size()));
+        let scratch = self
+            .scratch_memory
+            .as_ref()
+            .map(|memory| (memory.base_addr(), memory.mem_size()));
+        (snapshot, scratch)
+    }
+}

src/hyperlight_host/src/lib.rs

@@ -89,6 +89,8 @@ pub use hypervisor::virtual_machine::is_hypervisor_present;
 /// A sandbox that can call be used to make multiple calls to guest functions,
 /// and otherwise reused multiple times
 pub use sandbox::MultiUseSandbox;
+/// The lifecycle state of a [`MultiUseSandbox`].
+pub use sandbox::SandboxStatus;
 /// The re-export for the `UninitializedSandbox` type
 pub use sandbox::UninitializedSandbox;
 /// A collection of host functions that can be supplied to a sandbox