From b188a588bc651f4244ea5b0ae8f83f23f41c8d96 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 29 Nov 2023 07:46:18 -0800 Subject: [PATCH 1/2] On 32-bit x86, get the vsyscall address from the AUX vector On 32-bit x86, get the vsyscall address from the AT_SYSINFO AUX vector entry, rather than looking it up in the vDSO. This avoids the need to link in all the vDSO code if it isn't otherwise needed. And, it's simpler, avoiding the need for the `rustix_int_0x80` function. --- src/backend/linux_raw/arch/mod.rs | 6 +- src/backend/linux_raw/arch/x86.rs | 2 +- src/backend/linux_raw/mod.rs | 6 +- src/backend/linux_raw/param/auxv.rs | 110 ++++++++++--- src/backend/linux_raw/param/init.rs | 12 ++ src/backend/linux_raw/param/libc_auxv.rs | 32 ++++ src/backend/linux_raw/vdso_wrappers.rs | 199 ----------------------- src/backend/linux_raw/x86_vsyscall.rs | 103 ++++++++++++ 8 files changed, 240 insertions(+), 230 deletions(-) create mode 100644 src/backend/linux_raw/x86_vsyscall.rs diff --git a/src/backend/linux_raw/arch/mod.rs b/src/backend/linux_raw/arch/mod.rs index ac9e25fa7..12c82be18 100644 --- a/src/backend/linux_raw/arch/mod.rs +++ b/src/backend/linux_raw/arch/mod.rs @@ -51,11 +51,11 @@ pub(in crate::backend) mod asm; ))] pub(in crate::backend) use self::asm as choose; -// On 32-bit x86, use vDSO wrappers for all syscalls. We could use the -// architecture syscall instruction (`int 0x80`), but the vDSO kernel_vsyscall +// On 32-bit x86, use the kernel_vsyscall mechanism for syscalls. We could use +// the architecture syscall instruction (`int 0x80`), but the kernel_vsyscall // mechanism is much faster. #[cfg(target_arch = "x86")] -pub(in crate::backend) use super::vdso_wrappers::x86_via_vdso as choose; +pub(in crate::backend) use super::x86_vsyscall as choose; // This would be the code for always using `int 0x80` on 32-bit x86. //#[cfg(target_arch = "x86")] diff --git a/src/backend/linux_raw/arch/x86.rs b/src/backend/linux_raw/arch/x86.rs index e789181cc..2429659e5 100644 --- a/src/backend/linux_raw/arch/x86.rs +++ b/src/backend/linux_raw/arch/x86.rs @@ -15,7 +15,7 @@ use crate::backend::reg::{ ArgReg, FromAsm, RetReg, SyscallNumber, ToAsm, A0, A1, A2, A3, A4, A5, R0, }; -use crate::backend::vdso_wrappers::SyscallType; +use crate::backend::x86_vsyscall::SyscallType; use core::arch::asm; #[inline] diff --git a/src/backend/linux_raw/mod.rs b/src/backend/linux_raw/mod.rs index 0d4e5332d..335507f91 100644 --- a/src/backend/linux_raw/mod.rs +++ b/src/backend/linux_raw/mod.rs @@ -18,10 +18,12 @@ mod arch; mod conv; mod reg; -#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))] +#[cfg(any(feature = "time", feature = "process"))] mod vdso; -#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))] +#[cfg(any(feature = "time", feature = "process"))] mod vdso_wrappers; +#[cfg(target_arch = "x86")] +mod x86_vsyscall; #[cfg(feature = "event")] pub(crate) mod event; diff --git a/src/backend/linux_raw/param/auxv.rs b/src/backend/linux_raw/param/auxv.rs index 59c7a17ad..e8638394c 100644 --- a/src/backend/linux_raw/param/auxv.rs +++ b/src/backend/linux_raw/param/auxv.rs @@ -21,6 +21,8 @@ use core::sync::atomic::AtomicU8; use core::sync::atomic::Ordering::Relaxed; use core::sync::atomic::{AtomicPtr, AtomicUsize}; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use linux_raw_sys::general::AT_SYSINFO; use linux_raw_sys::general::{ AT_BASE, AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR, @@ -38,8 +40,12 @@ pub(crate) fn page_size() -> usize { let mut page_size = PAGE_SIZE.load(Relaxed); if page_size == 0 { - init_auxv(); - page_size = PAGE_SIZE.load(Relaxed); + #[cold] + fn compute_page_size() -> usize { + init_auxv(); + PAGE_SIZE.load(Relaxed) + } + page_size = compute_page_size(); } page_size @@ -51,8 +57,12 @@ pub(crate) fn clock_ticks_per_second() -> u64 { let mut ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed); if ticks == 0 { - init_auxv(); - ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed); + #[cold] + fn compute_clock_ticks_per_second() -> usize { + init_auxv(); + CLOCK_TICKS_PER_SECOND.load(Relaxed) + } + ticks = compute_clock_ticks_per_second(); } ticks as u64 @@ -65,9 +75,12 @@ pub(crate) fn linux_hwcap() -> (usize, usize) { let mut hwcap2 = HWCAP2.load(Relaxed); if hwcap == 0 || hwcap2 == 0 { - init_auxv(); - hwcap = HWCAP.load(Relaxed); - hwcap2 = HWCAP2.load(Relaxed); + #[cold] + fn compute_linux_hwcap() -> (usize, usize) { + init_auxv(); + (HWCAP.load(Relaxed), HWCAP2.load(Relaxed)) + } + (hwcap, hwcap2) = compute_linux_hwcap(); } (hwcap, hwcap2) @@ -92,8 +105,12 @@ pub(crate) fn linux_execfn() -> &'static CStr { let mut execfn = EXECFN.load(Relaxed); if execfn.is_null() { - init_auxv(); - execfn = EXECFN.load(Relaxed); + #[cold] + fn compute_linux_execfn() -> *mut c::c_char { + init_auxv(); + EXECFN.load(Relaxed) + } + execfn = compute_linux_execfn(); } // SAFETY: We assume the `AT_EXECFN` value provided by the kernel is a @@ -108,8 +125,12 @@ pub(crate) fn linux_secure() -> bool { // 0 means not initialized yet. if secure == 0 { - init_auxv(); - secure = SECURE.load(Relaxed); + #[cold] + fn compute_linux_secure() -> u8 { + init_auxv(); + SECURE.load(Relaxed) + } + secure = compute_linux_secure(); } // 0 means not present. Libc `getauxval(AT_SECURE)` would return 0. @@ -125,11 +146,13 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) { let mut phent = PHENT.load(Relaxed); let mut phnum = PHNUM.load(Relaxed); - if phdr.is_null() || phnum == 0 { - init_auxv(); - phdr = PHDR.load(Relaxed); - phent = PHENT.load(Relaxed); - phnum = PHNUM.load(Relaxed); + if phdr.is_null() || phent == 0 || phnum == 0 { + #[cold] + fn compute_exe_phdrs() -> (*mut Elf_Phdr, usize, usize) { + init_auxv(); + (PHDR.load(Relaxed), PHENT.load(Relaxed), PHNUM.load(Relaxed)) + } + (phdr, phent, phnum) = compute_exe_phdrs(); } (phdr.cast(), phent, phnum) @@ -145,12 +168,16 @@ pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr { let mut ehdr = SYSINFO_EHDR.load(Relaxed); if ehdr.is_null() { - // Use `maybe_init_auxv` to to read the aux vectors if it can, but do - // nothing if it can't. If it can't, then we'll get a null pointer - // here, which our callers are prepared to deal with. - maybe_init_auxv(); + #[cold] + fn compute_sysinfo_ehdr() -> *mut Elf_Ehdr { + // Use `maybe_init_auxv` to to read the aux vectors if it can, but do + // nothing if it can't. If it can't, then we'll get a null pointer + // here, which our callers are prepared to deal with. + maybe_init_auxv(); + SYSINFO_EHDR.load(Relaxed) + } - ehdr = SYSINFO_EHDR.load(Relaxed); + ehdr = compute_sysinfo_ehdr(); } ehdr @@ -162,8 +189,12 @@ pub(crate) fn entry() -> usize { let mut entry = ENTRY.load(Relaxed); if entry == 0 { - init_auxv(); - entry = ENTRY.load(Relaxed); + #[cold] + fn compute_entry() -> usize { + init_auxv(); + ENTRY.load(Relaxed) + } + entry = compute_entry(); } entry @@ -175,13 +206,34 @@ pub(crate) fn random() -> *const [u8; 16] { let mut random = RANDOM.load(Relaxed); if random.is_null() { - init_auxv(); - random = RANDOM.load(Relaxed); + #[cold] + fn compute_random() -> *mut [u8; 16] { + init_auxv(); + RANDOM.load(Relaxed) + } + random = compute_random(); } random } +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c::c_void { + let mut vsyscall = VSYSCALL.load(Relaxed); + + if vsyscall.is_null() { + #[cold] + fn compute_vsyscall() -> *const c::c_void { + init_auxv(); + VSYSCALL.load(Relaxed) + } + vsyscall = compute_vsyscall(); + } + + vsyscall +} + static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); static CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0); static HWCAP: AtomicUsize = AtomicUsize::new(0); @@ -201,6 +253,8 @@ static PHNUM: AtomicUsize = AtomicUsize::new(0); static ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(null_mut()); +#[cfg(feature = "x86")] +static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); const PR_GET_AUXV: c::c_int = 0x4155_5856; @@ -389,6 +443,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti let mut egid = None; #[cfg(feature = "runtime")] let mut random = null_mut(); + #[cfg(target_arch = "x86")] + let mut vsyscall = null_mut(); for Elf_auxv_t { a_type, a_val } in aux_iter { match a_type as _ { @@ -428,6 +484,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti AT_ENTRY => entry = a_val as usize, #[cfg(feature = "runtime")] AT_RANDOM => random = check_raw_pointer::<[u8; 16]>(a_val as *mut _)?.as_ptr(), + #[cfg(target_arch = "x86")] + AT_SYSINFO => vsyscall = a_val.cast(), AT_NULL => break, _ => (), @@ -464,6 +522,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti ENTRY.store(entry, Relaxed); #[cfg(feature = "runtime")] RANDOM.store(random, Relaxed); + #[cfg(target_arch = "x86")] + VSYSCALL.store(vsyscall, Relaxed); Some(()) } diff --git a/src/backend/linux_raw/param/init.rs b/src/backend/linux_raw/param/init.rs index 615f177a2..6da7cda2e 100644 --- a/src/backend/linux_raw/param/init.rs +++ b/src/backend/linux_raw/param/init.rs @@ -14,6 +14,8 @@ use core::ptr::{null_mut, read, NonNull}; use core::sync::atomic::AtomicBool; use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use linux_raw_sys::general::AT_SYSINFO; use linux_raw_sys::general::{ AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR, }; @@ -96,6 +98,12 @@ pub(crate) fn random() -> *const [u8; 16] { unsafe { RANDOM.load(Ordering::Relaxed) } } +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c_void { + unsafe { VSYSCALL.load(Ordering::Relaxed) } +} + static mut PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); static mut CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0); static mut HWCAP: AtomicUsize = AtomicUsize::new(0); @@ -118,6 +126,8 @@ static mut PHNUM: AtomicUsize = AtomicUsize::new(0); static mut ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static mut RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(NonNull::dangling().as_ptr()); +#[cfg(target_arch = "x86")] +static mut VSYSCALL: AtomicPtr = AtomicPtr::new(NonNull::dangling().as_ptr()); /// When "use-explicitly-provided-auxv" is enabled, we export a function to be /// called during initialization, and passed a pointer to the original @@ -170,6 +180,8 @@ unsafe fn init_from_auxp(mut auxp: *const Elf_auxv_t) { AT_ENTRY => ENTRY.store(a_val as usize, Ordering::Relaxed), #[cfg(feature = "runtime")] AT_RANDOM => RANDOM.store(a_val.cast::<[u8; 16]>(), Ordering::Relaxed), + #[cfg(feature = "x86")] + AT_SYSINFO => VSYSCALL.store(a_val.cast::(), Ordering::Relaxed), AT_NULL => break, _ => (), diff --git a/src/backend/linux_raw/param/libc_auxv.rs b/src/backend/linux_raw/param/libc_auxv.rs index c92e62a28..82128c055 100644 --- a/src/backend/linux_raw/param/libc_auxv.rs +++ b/src/backend/linux_raw/param/libc_auxv.rs @@ -11,6 +11,11 @@ use crate::ffi::CStr; #[cfg(not(feature = "runtime"))] use core::ptr::null; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use { + core::ffi::c_void, core::ptr::null_mut, core::sync::atomic::AtomicPtr, + core::sync::atomic::Ordering::Relaxed, +}; // `getauxval` wasn't supported in glibc until 2.16. Also this lets us use // `*mut` as the return type to preserve strict provenance. @@ -38,6 +43,8 @@ const AT_RANDOM: c::c_ulong = 25; const AT_HWCAP2: c::c_ulong = 26; const AT_SECURE: c::c_ulong = 23; const AT_EXECFN: c::c_ulong = 31; +#[cfg(target_arch = "x86")] +const AT_SYSINFO: c::c_ulong = 32; const AT_SYSINFO_EHDR: c::c_ulong = 33; // Declare `sysconf` ourselves so that we don't depend on all of libc just for @@ -72,6 +79,9 @@ fn test_abi() { const_assert_eq!(self::AT_ENTRY, ::libc::AT_ENTRY); #[cfg(feature = "runtime")] const_assert_eq!(self::AT_RANDOM, ::libc::AT_RANDOM); + // TODO: Upstream x86's `AT_SYSINFO` to libc. + #[cfg(target_arch = "x86")] + const_assert_eq!(self::AT_SYSINFO, ::linux_raw_sys::general::AT_SYSINFO); } #[cfg(feature = "param")] @@ -192,3 +202,25 @@ pub(crate) fn entry() -> usize { pub(crate) fn random() -> *const [u8; 16] { unsafe { getauxval(AT_RANDOM) as *const [u8; 16] } } + +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c_void { + // We call this for every system call, so memoize the value. + static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); + + let mut vsyscall = VSYSCALL.load(Relaxed); + + if vsyscall.is_null() { + #[cold] + fn compute_vsyscall() -> *mut c_void { + let vsyscall = unsafe { getauxval(AT_SYSINFO) } as *mut c_void; + VSYSCALL.store(vsyscall, Relaxed); + vsyscall + } + + vsyscall = compute_vsyscall(); + } + + vsyscall +} diff --git a/src/backend/linux_raw/vdso_wrappers.rs b/src/backend/linux_raw/vdso_wrappers.rs index 441738f8d..8256b21ae 100644 --- a/src/backend/linux_raw/vdso_wrappers.rs +++ b/src/backend/linux_raw/vdso_wrappers.rs @@ -9,11 +9,7 @@ //! functions. #![allow(unsafe_code)] -#[cfg(target_arch = "x86")] -use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; use super::vdso; -#[cfg(target_arch = "x86")] -use core::arch::global_asm; #[cfg(feature = "process")] #[cfg(any( target_arch = "x86_64", @@ -135,130 +131,6 @@ pub(crate) fn sched_getcpu() -> usize { } } -#[cfg(target_arch = "x86")] -pub(super) mod x86_via_vdso { - use super::{transmute, ArgReg, Relaxed, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; - use crate::backend::arch::asm; - - #[inline] - pub(in crate::backend) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall0(callee, nr) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall1<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall1(callee, nr, a0) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall1_noreturn<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - ) -> ! { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall1_noreturn(callee, nr, a0) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall2<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall2(callee, nr, a0, a1) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall3<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall3(callee, nr, a0, a1, a2) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall4<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall4(callee, nr, a0, a1, a2, a3) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall5<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - a4: ArgReg<'a, A4>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall6<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - a4: ArgReg<'a, A4>, - a5: ArgReg<'a, A5>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5) - } - - // With the indirect call, it isn't meaningful to do a separate - // `_readonly` optimization. - #[allow(unused_imports)] - pub(in crate::backend) use { - syscall0 as syscall0_readonly, syscall1 as syscall1_readonly, - syscall2 as syscall2_readonly, syscall3 as syscall3_readonly, - syscall4 as syscall4_readonly, syscall5 as syscall5_readonly, - syscall6 as syscall6_readonly, - }; -} - #[cfg(feature = "time")] type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_int; @@ -271,12 +143,6 @@ type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_in ))] type GetcpuType = unsafe extern "C" fn(*mut u32, *mut u32, *mut c_void) -> c::c_int; -/// The underlying syscall functions are only called from asm, using the -/// special syscall calling convention to pass arguments and return values, -/// which the signature here doesn't reflect. -#[cfg(target_arch = "x86")] -pub(super) type SyscallType = unsafe extern "C" fn(); - /// Initialize `CLOCK_GETTIME` and return its value. #[cfg(feature = "time")] #[cold] @@ -303,16 +169,6 @@ fn init_getcpu() -> GetcpuType { unsafe { transmute(GETCPU.load(Relaxed)) } } -/// Initialize `SYSCALL` and return its value. -#[cfg(target_arch = "x86")] -#[cold] -fn init_syscall() -> SyscallType { - init(); - // SAFETY: Load the function address from static storage that we just - // initialized. - unsafe { transmute(SYSCALL.load(Relaxed)) } -} - /// `AtomicPtr` can't hold a `fn` pointer, so we use a `*` pointer to this /// placeholder type, and cast it as needed. struct Function; @@ -326,8 +182,6 @@ static mut CLOCK_GETTIME: AtomicPtr = AtomicPtr::new(null_mut()); target_arch = "powerpc64" ))] static mut GETCPU: AtomicPtr = AtomicPtr::new(null_mut()); -#[cfg(target_arch = "x86")] -static mut SYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); #[cfg(feature = "time")] unsafe extern "C" fn rustix_clock_gettime_via_syscall( @@ -405,34 +259,6 @@ unsafe extern "C" fn rustix_getcpu_via_syscall( } } -#[cfg(target_arch = "x86")] -extern "C" { - /// A symbol pointing to an `int 0x80` instruction. This “function” is only - /// called from assembly, and only with the x86 syscall calling convention, - /// so its signature here is not its true signature. - /// - /// This extern block and the `global_asm!` below can be replaced with - /// `#[naked]` if it's stabilized. - fn rustix_int_0x80(); -} - -#[cfg(target_arch = "x86")] -global_asm!( - r#" - .section .text.rustix_int_0x80,"ax",@progbits - .p2align 4 - .weak rustix_int_0x80 - .hidden rustix_int_0x80 - .type rustix_int_0x80, @function -rustix_int_0x80: - .cfi_startproc - int 0x80 - ret - .cfi_endproc - .size rustix_int_0x80, .-rustix_int_0x80 -"# -); - fn minimal_init() { // SAFETY: Store default function addresses in static storage so that if we // end up making any system calls while we read the vDSO, they'll work. If @@ -468,18 +294,6 @@ fn minimal_init() { ) .ok(); } - - #[cfg(target_arch = "x86")] - { - SYSCALL - .compare_exchange( - null_mut(), - rustix_int_0x80 as *mut Function, - Relaxed, - Relaxed, - ) - .ok(); - } } } @@ -591,18 +405,5 @@ fn init() { } } } - - // On x86, also look up the vsyscall entry point. - #[cfg(target_arch = "x86")] - { - let ptr = vdso.sym(cstr!("LINUX_2.5"), cstr!("__kernel_vsyscall")); - assert!(!ptr.is_null()); - - // SAFETY: As above, store the computed function addresses in - // static storage. - unsafe { - SYSCALL.store(ptr.cast(), Relaxed); - } - } } } diff --git a/src/backend/linux_raw/x86_vsyscall.rs b/src/backend/linux_raw/x86_vsyscall.rs new file mode 100644 index 000000000..b013fdaa8 --- /dev/null +++ b/src/backend/linux_raw/x86_vsyscall.rs @@ -0,0 +1,103 @@ +//! Implement syscalls using the x86 vsyscall mechanism. +//! +//! # Safety +//! +//! Similar to syscalls.rs, this file performs raw system calls, and sometimes +//! passes them uninitialized memory buffers. +#![allow(unsafe_code)] + +use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; +use crate::backend::arch::asm; +use core::mem::transmute; + +#[inline] +pub(super) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall0(callee, nr) +} + +#[inline] +pub(super) unsafe fn syscall1<'a>(nr: SyscallNumber<'a>, a0: ArgReg<'a, A0>) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall1(callee, nr, a0) +} + +#[inline] +pub(super) unsafe fn syscall1_noreturn<'a>(nr: SyscallNumber<'a>, a0: ArgReg<'a, A0>) -> ! { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall1_noreturn(callee, nr, a0) +} + +#[inline] +pub(super) unsafe fn syscall2<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall2(callee, nr, a0, a1) +} + +#[inline] +pub(super) unsafe fn syscall3<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall3(callee, nr, a0, a1, a2) +} + +#[inline] +pub(super) unsafe fn syscall4<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall4(callee, nr, a0, a1, a2, a3) +} + +#[inline] +pub(super) unsafe fn syscall5<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, + a4: ArgReg<'a, A4>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4) +} + +#[inline] +pub(super) unsafe fn syscall6<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, + a4: ArgReg<'a, A4>, + a5: ArgReg<'a, A5>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5) +} + +// With the indirect call, it isn't meaningful to do a separate +// `_readonly` optimization. +#[allow(unused_imports)] +pub(super) use { + syscall0 as syscall0_readonly, syscall1 as syscall1_readonly, syscall2 as syscall2_readonly, + syscall3 as syscall3_readonly, syscall4 as syscall4_readonly, syscall5 as syscall5_readonly, + syscall6 as syscall6_readonly, +}; + +/// The underlying syscall functions are only called from asm, using the +/// special syscall calling convention to pass arguments and return values, +/// which the signature here doesn't reflect. +pub(super) type SyscallType = unsafe extern "C" fn(); From d8c14adb0e66467d4ca389addeb72dd34628450f Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 6 Dec 2023 15:07:12 -0800 Subject: [PATCH 2/2] Depend on glibc 2.17 and use getauxval unconditionally. --- README.md | 8 ++++ src/backend/libc/param/auxv.rs | 27 ++++--------- src/backend/linux_raw/param/auxv.rs | 4 +- src/backend/linux_raw/param/libc_auxv.rs | 50 ++---------------------- src/lib.rs | 12 ++---- tests/param/auxv.rs | 16 ++++---- 6 files changed, 32 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 9f437b1a4..722994a35 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,14 @@ oldest Linux version supported by: The specifics of this policy may change in the future, but we intend it to always reflect “very old” Linux versions. +## Minimum glibc Version + +On glibc platforms, rustix requires at least glibc 2.17. This is at most the +oldest glibc version supported by: + - [any current Rust target] +The specifics of this policy may change in the future, but we intend it to +always reflect “very old” glibc versions. + [MSRV]: #minimum-supported-rust-version-msrv [Rust 1.63]: https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html [any current Rust target]: https://doc.rust-lang.org/nightly/rustc/platform-support.html diff --git a/src/backend/libc/param/auxv.rs b/src/backend/libc/param/auxv.rs index e53767583..5b1e3359a 100644 --- a/src/backend/libc/param/auxv.rs +++ b/src/backend/libc/param/auxv.rs @@ -5,12 +5,9 @@ use crate::backend::c; ))] use crate::ffi::CStr; -// `getauxval` wasn't supported in glibc until 2.16. -#[cfg(any( - all(target_os = "android", target_pointer_width = "64"), - target_os = "linux", -))] -weak!(fn getauxval(c::c_ulong) -> *mut c::c_void); +extern "C" { + fn getauxval(type_: c::c_ulong) -> *mut c::c_void; +} #[inline] pub(crate) fn page_size() -> usize { @@ -29,14 +26,10 @@ pub(crate) fn clock_ticks_per_second() -> u64 { ))] #[inline] pub(crate) fn linux_hwcap() -> (usize, usize) { - if let Some(libc_getauxval) = getauxval.get() { - unsafe { - let hwcap = libc_getauxval(c::AT_HWCAP) as usize; - let hwcap2 = libc_getauxval(c::AT_HWCAP2) as usize; - (hwcap, hwcap2) - } - } else { - (0, 0) + unsafe { + let hwcap = getauxval(c::AT_HWCAP) as usize; + let hwcap2 = getauxval(c::AT_HWCAP2) as usize; + (hwcap, hwcap2) } } @@ -61,9 +54,5 @@ pub(crate) fn linux_minsigstksz() -> usize { ))] #[inline] pub(crate) fn linux_execfn() -> &'static CStr { - if let Some(libc_getauxval) = getauxval.get() { - unsafe { CStr::from_ptr(libc_getauxval(c::AT_EXECFN).cast()) } - } else { - cstr!("") - } + unsafe { CStr::from_ptr(getauxval(c::AT_EXECFN).cast()) } } diff --git a/src/backend/linux_raw/param/auxv.rs b/src/backend/linux_raw/param/auxv.rs index e8638394c..92ff86c6c 100644 --- a/src/backend/linux_raw/param/auxv.rs +++ b/src/backend/linux_raw/param/auxv.rs @@ -224,7 +224,7 @@ pub(crate) fn vsyscall() -> *const c::c_void { if vsyscall.is_null() { #[cold] - fn compute_vsyscall() -> *const c::c_void { + fn compute_vsyscall() -> *mut c::c_void { init_auxv(); VSYSCALL.load(Relaxed) } @@ -253,7 +253,7 @@ static PHNUM: AtomicUsize = AtomicUsize::new(0); static ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(null_mut()); -#[cfg(feature = "x86")] +#[cfg(target_arch = "x86")] static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); const PR_GET_AUXV: c::c_int = 0x4155_5856; diff --git a/src/backend/linux_raw/param/libc_auxv.rs b/src/backend/linux_raw/param/libc_auxv.rs index 82128c055..1b786541f 100644 --- a/src/backend/linux_raw/param/libc_auxv.rs +++ b/src/backend/linux_raw/param/libc_auxv.rs @@ -17,14 +17,6 @@ use { core::sync::atomic::Ordering::Relaxed, }; -// `getauxval` wasn't supported in glibc until 2.16. Also this lets us use -// `*mut` as the return type to preserve strict provenance. -#[cfg(not(feature = "runtime"))] -weak!(fn getauxval(c::c_ulong) -> *mut c::c_void); - -// With the "runtime" feature, go ahead and depend on `getauxval` existing so -// that we never fail. -#[cfg(feature = "runtime")] extern "C" { fn getauxval(type_: c::c_ulong) -> *mut c::c_void; } @@ -99,18 +91,6 @@ pub(crate) fn clock_ticks_per_second() -> u64 { #[cfg(feature = "param")] #[inline] pub(crate) fn linux_hwcap() -> (usize, usize) { - #[cfg(not(feature = "runtime"))] - unsafe { - if let Some(libc_getauxval) = getauxval.get() { - let hwcap = libc_getauxval(AT_HWCAP) as usize; - let hwcap2 = libc_getauxval(AT_HWCAP2) as usize; - (hwcap, hwcap2) - } else { - (0, 0) - } - } - - #[cfg(feature = "runtime")] unsafe { let hwcap = getauxval(AT_HWCAP) as usize; let hwcap2 = getauxval(AT_HWCAP2) as usize; @@ -140,19 +120,7 @@ pub(crate) fn linux_minsigstksz() -> usize { #[cfg(feature = "param")] #[inline] pub(crate) fn linux_execfn() -> &'static CStr { - #[cfg(not(feature = "runtime"))] - unsafe { - if let Some(libc_getauxval) = getauxval.get() { - CStr::from_ptr(libc_getauxval(AT_EXECFN).cast()) - } else { - cstr!("") - } - } - - #[cfg(feature = "runtime")] - unsafe { - CStr::from_ptr(getauxval(AT_EXECFN).cast()) - } + unsafe { CStr::from_ptr(getauxval(AT_EXECFN).cast()) } } #[cfg(feature = "runtime")] @@ -176,19 +144,7 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) { /// if we don't see it, this function returns a null pointer. #[inline] pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr { - #[cfg(not(feature = "runtime"))] - unsafe { - if let Some(libc_getauxval) = getauxval.get() { - libc_getauxval(AT_SYSINFO_EHDR) as *const Elf_Ehdr - } else { - null() - } - } - - #[cfg(feature = "runtime")] - unsafe { - getauxval(AT_SYSINFO_EHDR) as *const Elf_Ehdr - } + unsafe { getauxval(AT_SYSINFO_EHDR) as *const Elf_Ehdr } } #[cfg(feature = "runtime")] @@ -214,7 +170,7 @@ pub(crate) fn vsyscall() -> *const c_void { if vsyscall.is_null() { #[cold] fn compute_vsyscall() -> *mut c_void { - let vsyscall = unsafe { getauxval(AT_SYSINFO) } as *mut c_void; + let vsyscall = unsafe { getauxval(AT_SYSINFO) as *mut c_void }; VSYSCALL.store(vsyscall, Relaxed); vsyscall } diff --git a/src/lib.rs b/src/lib.rs index 51fbdbaf7..623810f39 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -157,15 +157,9 @@ pub(crate) mod check_types; #[macro_use] pub(crate) mod bitcast; -// linux_raw: Weak symbols are used by the use-libc-auxv feature for -// glibc 2.15 support. -// -// libc: Weak symbols are used to call various functions available in some -// versions of libc and not others. -#[cfg(any( - all(linux_raw, feature = "use-libc-auxv"), - all(libc, not(any(windows, target_os = "espidf", target_os = "wasi"))) -))] +// Weak symbols are used to call various functions available in some versions +// of libc and not others. +#[cfg(all(libc, not(any(windows, target_os = "espidf", target_os = "wasi"))))] #[macro_use] mod weak; diff --git a/tests/param/auxv.rs b/tests/param/auxv.rs index 7806fa357..99e23860f 100644 --- a/tests/param/auxv.rs +++ b/tests/param/auxv.rs @@ -27,17 +27,17 @@ fn test_clock_ticks_per_second() { ))] #[test] fn test_linux_hwcap() { - weak!(fn getauxval(libc::c_ulong) -> libc::c_ulong); + extern "C" { + fn getauxval(type_: c::c_ulong) -> *mut c::c_void; + } - if let Some(libc_getauxval) = getauxval.get() { - let (_hwcap, hwcap2) = linux_hwcap(); + let (_hwcap, hwcap2) = linux_hwcap(); - // glibc seems to return a different value than `LD_SHOW_AUXV=1` reports. - #[cfg(not(target_env = "gnu"))] - assert_eq!(_hwcap, unsafe { libc_getauxval(libc::AT_HWCAP) } as usize); + // glibc seems to return a different value than `LD_SHOW_AUXV=1` reports. + #[cfg(not(target_env = "gnu"))] + assert_eq!(_hwcap, unsafe { getauxval(libc::AT_HWCAP) } as usize); - assert_eq!(hwcap2, unsafe { libc_getauxval(libc::AT_HWCAP2) } as usize); - } + assert_eq!(hwcap2, unsafe { getauxval(libc::AT_HWCAP2) } as usize); } #[cfg(any(