From aa9e8d2a275917f2a84bc14c5f23cf6a331412cd Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 14 Apr 2026 10:57:27 -0700 Subject: [PATCH] [kernel] E: Batch page allocation in ELF loader Restructure the ELF segment loader to batch-allocate contiguous page runs instead of allocating one page at a time via alloc_upages(1). Pages within each segment are grouped by their clear requirement (data pages that will be overwritten vs BSS pages that must be zeroed) and each contiguous run is allocated with a single alloc_upages(nframes) call. This reduces the number of frame-allocator and page-table operations from O(pages) to O(runs), typically 2-3 per segment (one for the data region and one for the BSS region). To support large batch sizes without exhausting the 128 KB kernel slab, alloc_upages is refactored to process frames in fixed-size chunks (MAX_CHUNK = 32). Each chunk allocates a small Vec that is dropped before the next chunk begins. A rollback helper and a free_remaining_frames helper ensure that both mapped pages and unprocessed frames are properly cleaned up on error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/kernel/src/mm/elf.rs | 137 +++++++++++++++++------------- src/kernel/src/mm/virt/manager.rs | 127 +++++++++++++++++---------- 2 files changed, 163 insertions(+), 101 deletions(-) diff --git a/src/kernel/src/mm/elf.rs b/src/kernel/src/mm/elf.rs index e655336a52..a637abdf61 100644 --- a/src/kernel/src/mm/elf.rs +++ b/src/kernel/src/mm/elf.rs @@ -261,31 +261,42 @@ fn do_elf32_load( let phys_addr_end: usize = phys_addr_base + phdr.p_filesz as usize; - // Load segment page by page. + // Load segment: batch-allocate contiguous page runs, then copy data. debug!( "loading segment (virt_addr_base={:#x}, virt_addr_end={:#x}, phys_addr_base={:#x}, \ phys_addr_end={:#x}, access={:?})", virt_addr_base, virt_addr_end, phys_addr_base, phys_addr_end, access ); - for vaddr in (virt_addr_base..virt_addr_end).step_by(mem::PAGE_SIZE) { - let vaddr: VirtualAddress = VirtualAddress::new(vaddr); - - // Check if address lies in user space. - if vaddr < USER_BASE { - let reason: &str = "invalid load address"; - error!("{reason}"); - return Err(Error::new(ErrorCode::BadFile, reason)); - } - - let vaddr: PageAligned = PageAligned::from_address(vaddr)?; - - // Check if we should perform the allocation. - if !dry_run { - let page_addr: usize = vaddr.into_raw_value(); - - // Scan prior segment ranges to detect overlap and compute the merged - // permission from all segments that cover this page. + // Phase 1: Batch-allocate pages. + // + // Instead of allocating one page at a time, collect contiguous runs of + // unmapped pages that share the same `clear` requirement and allocate + // each run with a single `alloc_upages(nframes)` call. This reduces + // the number of frame-allocator + page-table operations from O(pages) + // separate calls to O(runs), which is typically 2-3 per segment (one + // for the data region and one for the BSS region). + if !dry_run { + let mut run_start: Option = None; + let mut run_clear: bool = false; + + // Helper: flush a pending run of contiguous pages. + let flush_run = + |mm: &mut VirtMemoryManager, + vmem: &mut Vmem, + start: usize, + end: usize, + clear: bool, + access: AccessPermission| + -> Result<(), Error> { + let nframes: usize = (end - start) / mem::PAGE_SIZE; + let start_vaddr: PageAligned = + PageAligned::from_raw_value(start)?; + mm.alloc_upages(vmem, start_vaddr, nframes, access, clear) + }; + + for page_addr in (virt_addr_base..virt_addr_end).step_by(mem::PAGE_SIZE) { + // Scan prior segment ranges to detect overlap. let mut already_mapped: bool = false; let mut merged: AccessPermission = access; for &(start, end, prev_access) in loaded_ranges.iter().take(loaded_count) { @@ -296,52 +307,62 @@ fn do_elf32_load( } if already_mapped { - // Page already mapped by a prior segment — apply merged permissions - // to accommodate all segments sharing this page. + // Flush any pending run before handling the overlap. + if let Some(start) = run_start.take() { + flush_run(mm, vmem, start, page_addr, run_clear, access)?; + } + let vaddr: PageAligned = + PageAligned::from_raw_value(page_addr)?; mm.ctrl_upage(vmem, vaddr, merged)?; } else { - // Only clear pages that will NOT be fully overwritten by segment data. - // A page is fully covered when the segment data for this page spans - // the entire PAGE_SIZE bytes; in that case clearing is redundant. - - // Start of the physical/source-backed data for this page. - let page_offset_in_segment: usize = vaddr.into_raw_value() - virt_addr_base; - let page_phys_addr: usize = - match phys_addr_base.checked_add(page_offset_in_segment) { - Some(addr) => addr, - None => { - let reason: &str = "invalid physical address"; - error!("{reason}"); - return Err(Error::new(ErrorCode::BadFile, reason)); - }, - }; - // One-past-the-end if the full page were backed by segment data. - let page_phys_addr_end: usize = match page_phys_addr.checked_add(mem::PAGE_SIZE) - { - Some(end) => end, + // Determine whether this page needs clearing. + let page_offset: usize = page_addr - virt_addr_base; + let page_phys: usize = phys_addr_base.checked_add(page_offset).ok_or_else( + || Error::new(ErrorCode::BadFile, "invalid physical address"), + )?; + let page_phys_end: usize = + page_phys.checked_add(mem::PAGE_SIZE).ok_or_else(|| { + Error::new(ErrorCode::BadFile, "invalid physical address range") + })?; + let needs_clear: bool = + page_phys >= phys_addr_end || page_phys_end > phys_addr_end; + + match run_start { + Some(_) if needs_clear == run_clear => { + // Extend the current run (same clear requirement). + }, + Some(start) => { + // Clear requirement changed -- flush and start a new run. + flush_run(mm, vmem, start, page_addr, run_clear, access)?; + run_start = Some(page_addr); + run_clear = needs_clear; + }, None => { - let reason: &str = "invalid physical address range"; - error!("{reason}"); - return Err(Error::new(ErrorCode::BadFile, reason)); + run_start = Some(page_addr); + run_clear = needs_clear; }, - }; - - // Page is entirely beyond segment data (pure BSS) — must be zeroed. - let page_lies_in_bss: bool = page_phys_addr >= phys_addr_end; - // Page straddles the segment-data/BSS boundary — trailing bytes must - // be zeroed. - let page_is_partially_covered: bool = - page_phys_addr < phys_addr_end && page_phys_addr_end > phys_addr_end; - mm.alloc_upages( - vmem, - vaddr, - 1, - access, - page_lies_in_bss || page_is_partially_covered, - )?; + } } } + // Flush the final run. + if let Some(start) = run_start { + flush_run(mm, vmem, start, virt_addr_end, run_clear, access)?; + } + } + + // Phase 2: Copy segment data and update last_address. + for vaddr in (virt_addr_base..virt_addr_end).step_by(mem::PAGE_SIZE) { + let vaddr: VirtualAddress = VirtualAddress::new(vaddr); + + if vaddr < USER_BASE { + let reason: &str = "invalid load address"; + error!("{reason}"); + return Err(Error::new(ErrorCode::BadFile, reason)); + } + + let vaddr: PageAligned = PageAligned::from_address(vaddr)?; + // Update last address. if vaddr.into_raw_value() + mem::PAGE_SIZE > last_address { last_address = vaddr.into_raw_value() + mem::PAGE_SIZE; diff --git a/src/kernel/src/mm/virt/manager.rs b/src/kernel/src/mm/virt/manager.rs index 309acf46ed..0165e64656 100644 --- a/src/kernel/src/mm/virt/manager.rs +++ b/src/kernel/src/mm/virt/manager.rs @@ -310,60 +310,101 @@ impl VirtMemoryManager { Ok(page_table) }; - let uframes: Vec = match self.physman.try_borrow_mut() { - Ok(mut physman) => physman.alloc_many_user_frames(nframes)?, - Err(_) => { - let reason: &str = "failed to borrow physical memory manager"; - error!("{reason}"); - return Err(Error::new(ErrorCode::ResourceBusy, reason)); - }, - }; + // Process frames in chunks to bound kernel heap usage. Each chunk + // allocates a small Vec that is dropped before the next chunk begins, + // preventing large batch requests from exhausting the kernel slab. + const MAX_CHUNK: usize = 32; let start_vaddr: PageAligned = vaddr; - let mut mapped_count: usize = 0; - let mut map_error: Result<(), Error> = Ok(()); + let mut total_mapped: usize = 0; + let mut remaining: usize = nframes; + + while remaining > 0 { + let chunk_size: usize = core::cmp::min(remaining, MAX_CHUNK); + + let alloc_result = self + .physman + .try_borrow_mut() + .map_err(|_| { + Error::new( + ErrorCode::ResourceBusy, + "failed to borrow physical memory manager", + ) + }) + .and_then(|mut physman| physman.alloc_many_user_frames(chunk_size)); + + let uframes: Vec = match alloc_result { + Ok(frames) => frames, + Err(e) => { + error!("alloc_upages(): frame allocation failed ({e:?})"); + self.rollback_mapped_pages(vmem, start_vaddr, total_mapped); + return Err(e); + }, + }; - for uframe in uframes { - if let Err(e) = vmem.map(uframe, vaddr, access, &page_table_allocator) { - map_error = Err(e); - break; - } - mapped_count += 1; - if clear { - if let Err(e) = vmem.memset(vaddr, 0) { - map_error = Err(e); - break; + let mut uframes_iter = uframes.into_iter(); + for uframe in uframes_iter.by_ref() { + if let Err(e) = vmem.map(uframe, vaddr, access, &page_table_allocator) { + // Free remaining unprocessed frames to avoid leaking them. + self.free_remaining_frames(uframes_iter); + self.rollback_mapped_pages(vmem, start_vaddr, total_mapped); + return Err(e); + } + total_mapped += 1; + if clear { + if let Err(e) = vmem.memset(vaddr, 0) { + self.free_remaining_frames(uframes_iter); + self.rollback_mapped_pages(vmem, start_vaddr, total_mapped); + return Err(e); + } + } + match PageAligned::from_raw_value(vaddr.into_raw_value() + mem::PAGE_SIZE) { + Ok(next) => vaddr = next, + Err(e) => { + self.free_remaining_frames(uframes_iter); + self.rollback_mapped_pages(vmem, start_vaddr, total_mapped); + return Err(e); + }, } } - match PageAligned::from_raw_value(vaddr.into_raw_value() + mem::PAGE_SIZE) { - Ok(next) => vaddr = next, - Err(e) => { - map_error = Err(e); - break; - }, + // Vec is dropped here, freeing kernel heap before the next chunk. + + remaining -= chunk_size; + } + + Ok(()) + } + + /// Rollback helper: unmap pages that were successfully mapped starting from + /// `start_vaddr` for `count` pages. + fn rollback_mapped_pages( + &mut self, + vmem: &mut Vmem, + start_vaddr: PageAligned, + count: usize, + ) { + let mut addr: PageAligned = start_vaddr; + for _ in 0..count { + if let Err(re) = self.try_unmap_upage(vmem, addr) { + warn!("alloc_upages(): rollback failed (vaddr={addr:?}, error={re:?})"); } + addr = match PageAligned::from_raw_value(addr.into_raw_value() + mem::PAGE_SIZE) { + Ok(next) => next, + Err(_) => break, + }; } + } - if let Err(e) = map_error { - // Rollback: unmap all pages that were successfully mapped. - let mut rollback_addr: PageAligned = start_vaddr; - for _ in 0..mapped_count { - if let Err(re) = self.try_unmap_upage(vmem, rollback_addr) { - warn!( - "alloc_upages(): rollback failed (vaddr={rollback_addr:?}, error={re:?})" - ); + /// Free frames that were allocated but not yet mapped, to avoid resource leaks + /// when a map or memset operation fails mid-chunk. + fn free_remaining_frames(&mut self, remaining: impl Iterator) { + if let Ok(mut physman) = self.physman.try_borrow_mut() { + for frame in remaining { + if let Err(fe) = physman.free_user_frame(frame) { + warn!("alloc_upages(): failed to free remaining frame (error={fe:?})"); } - rollback_addr = match PageAligned::from_raw_value( - rollback_addr.into_raw_value() + mem::PAGE_SIZE, - ) { - Ok(next) => next, - Err(_) => break, - }; } - return Err(e); } - - Ok(()) } ///