diff --git a/src/arch/x86_64/kernel/interrupts.rs b/src/arch/x86_64/kernel/interrupts.rs index 0aff28c0aa..6988146920 100644 --- a/src/arch/x86_64/kernel/interrupts.rs +++ b/src/arch/x86_64/kernel/interrupts.rs @@ -158,7 +158,10 @@ pub(crate) fn install() { .set_stack_index(0); } - IRQ_NAMES.lock().insert(7, "FPU"); + let mut guard = IRQ_NAMES.lock(); + #[cfg(feature = "mman")] + guard.insert(14, "Page Fault"); + guard.insert(7, "FPU"); } pub(crate) fn install_handlers() { diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 109770afc4..eda4a4e728 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -279,8 +279,18 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( stack_frame: ExceptionStackFrame, error_code: PageFaultErrorCode, ) { + let addr = Cr2::read().unwrap(); + + #[cfg(feature = "mman")] + crate::arch::x86_64::kernel::core_local::increment_irq_counter(14); + #[cfg(feature = "mman")] + if crate::syscalls::mman::resolve_page_fault(addr.into()).is_ok() { + crate::kernel::apic::eoi(); + return; + } + error!("Page fault (#PF)!"); - error!("page_fault_linear_address = {:p}", Cr2::read().unwrap()); + error!("page_fault_linear_address = {addr:p}"); error!("error_code = {error_code:?}"); error!("fs = {:#X}", processor::readfs()); error!("gs = {:#X}", processor::readgs()); diff --git a/src/lib.rs b/src/lib.rs index 326d7db3ba..9ff1720ae2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ #![feature(allocator_api)] #![feature(linkage)] #![feature(linked_list_cursors)] +#![feature(linked_list_retain)] #![feature(map_try_insert)] #![feature(maybe_uninit_as_bytes)] #![feature(maybe_uninit_slice)] diff --git a/src/syscalls/mman.rs b/src/syscalls/mman.rs index ec1078c2b6..e9c672ef28 100644 --- a/src/syscalls/mman.rs +++ b/src/syscalls/mman.rs @@ -1,8 +1,12 @@ +use alloc::collections::LinkedList; use core::ffi::{c_int, c_void}; use align_address::Align; -use free_list::{PageLayout, PageRange}; -use memory_addresses::{PhysAddr, VirtAddr}; +use free_list::{PageLayout, PageRange, PageRangeSub}; +use hermit_sync::InterruptSpinMutex; +#[cfg(not(feature = "common-os"))] +use memory_addresses::PhysAddr; +use memory_addresses::VirtAddr; use crate::arch; #[cfg(target_arch = "x86_64")] @@ -10,10 +14,22 @@ use crate::arch::mm::paging::PageTableEntryFlagsExt; use crate::arch::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags}; use crate::mm::physicalmem::PHYSICAL_FREE_LIST; use crate::mm::virtualmem::KERNEL_FREE_LIST; +use crate::syscalls::Errno; + +#[derive(Debug, Copy, Clone)] +struct MemoryRegions { + /// Page range of the memory region. + pub page_range: PageRange, + /// The protection flags for the memory region. + pub prot_flags: MemoryProtection, +} + +static MEMORY_REGIONS: InterruptSpinMutex> = + InterruptSpinMutex::new(LinkedList::new()); bitflags! { #[repr(transparent)] - #[derive(Debug, Copy, Clone, Default)] + #[derive(Debug, Copy, Clone, Default, Eq, PartialEq)] pub struct MemoryProtection: u32 { /// Pages may not be accessed. const None = 0; @@ -32,59 +48,237 @@ bitflags! { #[unsafe(no_mangle)] pub extern "C" fn sys_mmap(size: usize, prot_flags: MemoryProtection, ret: &mut *mut u8) -> i32 { let size = size.align_up(BasePageSize::SIZE as usize); - let layout = PageLayout::from_size(size).unwrap(); - let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap(); - let virtual_address = VirtAddr::from(page_range.start()); - if prot_flags.is_empty() { + + if (*ret).is_null() { + let layout = PageLayout::from_size(size).unwrap(); + let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap(); + + let memory_region = MemoryRegions { + page_range, + prot_flags, + }; + + MEMORY_REGIONS.lock().push_back(memory_region); + + let virtual_address = VirtAddr::from(page_range.start()); + debug!("Mmap {virtual_address:X} ({size}) -> {prot_flags:?})"); + *ret = virtual_address.as_mut_ptr(); - return 0; - } - let frame_layout = PageLayout::from_size(size).unwrap(); - let frame_range = PHYSICAL_FREE_LIST.lock().allocate(frame_layout).unwrap(); - let physical_address = PhysAddr::from(frame_range.start()); - debug!("Mmap {physical_address:X} -> {virtual_address:X} ({size})"); - let count = size / BasePageSize::SIZE as usize; - let mut flags = PageTableEntryFlags::empty(); - flags.normal().writable(); - if prot_flags.contains(MemoryProtection::Write) { - flags.writable(); - } - if !prot_flags.contains(MemoryProtection::Exec) { - flags.execute_disable(); + 0 + } else { + let virtual_address = VirtAddr::new(*ret as u64); + let virtual_address = virtual_address.align_down(BasePageSize::SIZE); + let current_range = + PageRange::from_start_len(virtual_address.as_usize(), BasePageSize::SIZE as usize) + .unwrap(); + let mut memory_regions = MEMORY_REGIONS.lock(); + + for region in memory_regions.iter_mut() { + if region.page_range.contains(current_range) { + let page_range = + PageRange::from_start_len(virtual_address.as_usize(), size).unwrap(); + let sub_page_range = region.page_range - page_range; + let original_flags = region.prot_flags; + region.page_range = page_range; + region.prot_flags = prot_flags; + + match sub_page_range { + PageRangeSub::One(range) => { + let memory_region = MemoryRegions { + page_range: range, + prot_flags: original_flags, + }; + memory_regions.push_back(memory_region); + } + PageRangeSub::Two(range1, range2) => { + let memory_region = MemoryRegions { + page_range: range1, + prot_flags: original_flags, + }; + memory_regions.push_back(memory_region); + let memory_region = MemoryRegions { + page_range: range2, + prot_flags: original_flags, + }; + memory_regions.push_back(memory_region); + } + PageRangeSub::None => {} + } + + return 0; + } + } + + debug!( + "Try reserve memory region at {:X} with size {size}", + virtual_address.as_usize() + ); + let page_range = PageRange::from_start_len(*ret as usize, size).unwrap(); + if KERNEL_FREE_LIST.lock().allocate_at(page_range).is_ok() { + let memory_region = MemoryRegions { + page_range, + prot_flags, + }; + + MEMORY_REGIONS.lock().push_back(memory_region); + + 0 + } else { + error!( + "Unable to reserve memory region at {:X} with size {size}", + virtual_address.as_usize() + ); + -i32::from(Errno::Fault) + } } +} - arch::mm::paging::map::(virtual_address, physical_address, count, flags); +#[cfg(not(feature = "common-os"))] +pub(crate) fn resolve_page_fault(virtual_address: VirtAddr) -> Result<(), ()> { + debug!("Resolve page fault at {virtual_address:X}"); + let virtual_address = virtual_address.align_down(BasePageSize::SIZE); + let current_range = + PageRange::from_start_len(virtual_address.as_usize(), BasePageSize::SIZE as usize).unwrap(); + let mut memory_regions = MEMORY_REGIONS.lock(); - *ret = virtual_address.as_mut_ptr(); + for region in memory_regions.iter_mut() { + if region.page_range.contains(current_range) { + if region.prot_flags.is_empty() { + error!( + "Memory region {region:X?} has no protection flags set, cannot resolve page fault" + ); + return Err(()); + } - 0 + let frame_layout = + PageLayout::from_size(BasePageSize::SIZE.try_into().unwrap()).unwrap(); + let frame_range = PHYSICAL_FREE_LIST.lock().allocate(frame_layout).unwrap(); + let physical_address = PhysAddr::from(frame_range.start()); + + if !region.prot_flags.contains(MemoryProtection::Write) { + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable(); + + arch::mm::paging::map::(virtual_address, physical_address, 1, flags); + + let slice = unsafe { + alloc::slice::from_raw_parts_mut( + virtual_address.as_mut_ptr(), + BasePageSize::SIZE as usize / core::mem::size_of::(), + ) + }; + for byte in slice.iter_mut() { + *byte = 0u64; // Initialize the page to zero + } + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal(); + if region.prot_flags.contains(MemoryProtection::Write) { + flags.writable(); + } + if !region.prot_flags.contains(MemoryProtection::Exec) { + flags.execute_disable(); + } + arch::mm::paging::map::(virtual_address, physical_address, 1, flags); + + if region.prot_flags.contains(MemoryProtection::Write) { + let slice = unsafe { + alloc::slice::from_raw_parts_mut( + virtual_address.as_mut_ptr(), + BasePageSize::SIZE as usize / core::mem::size_of::(), + ) + }; + for byte in slice.iter_mut() { + *byte = 0u64; // Initialize the page to zero + } + } + + return Ok(()); + } + } + + error!( + "Don't find memory region for address {:X}!", + virtual_address.as_usize() + ); + + Err(()) } /// Unmaps memory at the specified `ptr` for `size` bytes. #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_munmap(ptr: *mut u8, size: usize) -> i32 { - let virtual_address = VirtAddr::from_ptr(ptr); let size = size.align_up(BasePageSize::SIZE as usize); + let virtual_address = VirtAddr::from_ptr(ptr); + let page_range = PageRange::from_start_len(virtual_address.as_usize(), size).unwrap(); + let mut memory_regions = MEMORY_REGIONS.lock(); - if let Some(physical_address) = arch::mm::paging::virtual_to_physical(virtual_address) { - arch::mm::paging::unmap::( - virtual_address, - size / BasePageSize::SIZE as usize, - ); - debug!("Unmapping {virtual_address:X} ({size}) -> {physical_address:X}"); + let mut sub_page_range = None; + let mut prot_flags = None; + memory_regions.retain(|region| { + if region.page_range.contains(page_range) { + sub_page_range = Some(region.page_range - page_range); + prot_flags = Some(region.prot_flags); - let range = PageRange::from_start_len(physical_address.as_u64() as usize, size).unwrap(); - if let Err(_err) = unsafe { PHYSICAL_FREE_LIST.lock().deallocate(range) } { - // FIXME: return EINVAL instead, once wasmtime can handle it - error!("Unable to deallocate {range:?}"); + false + } else { + true + } + }); + + if let Some(sub_page_range) = sub_page_range { + match sub_page_range { + PageRangeSub::One(range) => { + let memory_region = MemoryRegions { + page_range: range, + prot_flags: prot_flags.unwrap(), + }; + memory_regions.push_back(memory_region); + } + PageRangeSub::Two(range1, range2) => { + let memory_region1 = MemoryRegions { + page_range: range1, + prot_flags: prot_flags.unwrap(), + }; + memory_regions.push_back(memory_region1); + + let memory_region2 = MemoryRegions { + page_range: range2, + prot_flags: prot_flags.unwrap(), + }; + memory_regions.push_back(memory_region2); + } + PageRangeSub::None => {} } } - let range = PageRange::from_start_len(virtual_address.as_usize(), size).unwrap(); unsafe { - KERNEL_FREE_LIST.lock().deallocate(range).unwrap(); + KERNEL_FREE_LIST.lock().deallocate(page_range).unwrap(); + } + + for i in 0..(size / BasePageSize::SIZE as usize) { + if let Some(physical_address) = + arch::mm::paging::virtual_to_physical(virtual_address + i * BasePageSize::SIZE as usize) + { + arch::mm::paging::unmap::( + virtual_address + i * BasePageSize::SIZE as usize, + 1, + ); + debug!("Unmapping {virtual_address:X} ({size}) -> {physical_address:X}"); + + let range = PageRange::from_start_len( + physical_address.as_u64() as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); + if let Err(_err) = unsafe { PHYSICAL_FREE_LIST.lock().deallocate(range) } { + // FIXME: return EINVAL instead, once wasmtime can handle it + error!("Unable to deallocate {range:?}"); + } + } } 0 @@ -97,9 +291,64 @@ pub extern "C" fn sys_munmap(ptr: *mut u8, size: usize) -> i32 { #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_mprotect(ptr: *mut u8, size: usize, prot_flags: MemoryProtection) -> i32 { - let count = size / BasePageSize::SIZE as usize; + let size = size.align_up(BasePageSize::SIZE as usize); + let virtual_address = VirtAddr::from_ptr(ptr); + let virtual_address = virtual_address.align_down(BasePageSize::SIZE); + let current_range = PageRange::new( + virtual_address.as_usize(), + virtual_address.as_usize() + BasePageSize::SIZE as usize, + ) + .unwrap(); + let mut memory_regions = MEMORY_REGIONS.lock(); + + let mut found = false; + for region in memory_regions.iter_mut() { + if region.page_range.contains(current_range) { + let new_range = PageRange::from_start_len(virtual_address.as_usize(), size).unwrap(); + + let sub_range = region.page_range - new_range; + let original_flags = region.prot_flags; + found = true; + region.page_range = new_range; + region.prot_flags = prot_flags; + + match sub_range { + PageRangeSub::One(range) => { + let memory_region = MemoryRegions { + page_range: range, + prot_flags: original_flags, + }; + memory_regions.push_back(memory_region); + } + PageRangeSub::Two(range1, range2) => { + let memory_region = MemoryRegions { + page_range: range1, + prot_flags: original_flags, + }; + memory_regions.push_back(memory_region); + + let memory_region = MemoryRegions { + page_range: range2, + prot_flags: original_flags, + }; + memory_regions.push_back(memory_region); + } + PageRangeSub::None => {} + } + + break; + } + } + + if !found { + error!("No memory region found for {current_range:?}"); + return -i32::from(Errno::Nomem); + } + + drop(memory_regions); + let mut flags = PageTableEntryFlags::empty(); - flags.normal().writable(); + flags.normal(); if prot_flags.contains(MemoryProtection::Write) { flags.writable(); } @@ -107,19 +356,20 @@ pub extern "C" fn sys_mprotect(ptr: *mut u8, size: usize, prot_flags: MemoryProt flags.execute_disable(); } - let virtual_address = VirtAddr::from_ptr(ptr); - - debug!("Mprotect {virtual_address:X} ({size}) -> {prot_flags:?})"); - if let Some(physical_address) = arch::mm::paging::virtual_to_physical(virtual_address) { - arch::mm::paging::map::(virtual_address, physical_address, count, flags); - 0 - } else { - let frame_layout = PageLayout::from_size(size).unwrap(); - let frame_range = PHYSICAL_FREE_LIST.lock().allocate(frame_layout).unwrap(); - let physical_address = PhysAddr::from(frame_range.start()); - arch::mm::paging::map::(virtual_address, physical_address, count, flags); - 0 + debug!("mprotect {virtual_address:X} ({size}) -> {prot_flags:?})"); + for i in 0..(size / BasePageSize::SIZE as usize) { + if let Some(physical_address) = + arch::mm::paging::virtual_to_physical(virtual_address + i * BasePageSize::SIZE as usize) + { + arch::mm::paging::map::( + virtual_address + i * BasePageSize::SIZE as usize, + physical_address, + 1, + flags, + ); + } } + 0 } #[hermit_macro::system(errno)] diff --git a/src/syscalls/mod.rs b/src/syscalls/mod.rs index 7b8e015ad6..459c6de951 100644 --- a/src/syscalls/mod.rs +++ b/src/syscalls/mod.rs @@ -38,7 +38,7 @@ mod entropy; mod futex; pub(crate) mod interfaces; #[cfg(feature = "mman")] -mod mman; +pub(crate) mod mman; mod processor; #[cfg(feature = "newlib")] mod recmutex;