diff --git a/README.md b/README.md index 9f437b1a4..722994a35 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,14 @@ oldest Linux version supported by: The specifics of this policy may change in the future, but we intend it to always reflect “very old” Linux versions. +## Minimum glibc Version + +On glibc platforms, rustix requires at least glibc 2.17. This is at most the +oldest glibc version supported by: + - [any current Rust target] +The specifics of this policy may change in the future, but we intend it to +always reflect “very old” glibc versions. + [MSRV]: #minimum-supported-rust-version-msrv [Rust 1.63]: https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html [any current Rust target]: https://doc.rust-lang.org/nightly/rustc/platform-support.html diff --git a/src/backend/libc/param/auxv.rs b/src/backend/libc/param/auxv.rs index e53767583..5b1e3359a 100644 --- a/src/backend/libc/param/auxv.rs +++ b/src/backend/libc/param/auxv.rs @@ -5,12 +5,9 @@ use crate::backend::c; ))] use crate::ffi::CStr; -// `getauxval` wasn't supported in glibc until 2.16. -#[cfg(any( - all(target_os = "android", target_pointer_width = "64"), - target_os = "linux", -))] -weak!(fn getauxval(c::c_ulong) -> *mut c::c_void); +extern "C" { + fn getauxval(type_: c::c_ulong) -> *mut c::c_void; +} #[inline] pub(crate) fn page_size() -> usize { @@ -29,14 +26,10 @@ pub(crate) fn clock_ticks_per_second() -> u64 { ))] #[inline] pub(crate) fn linux_hwcap() -> (usize, usize) { - if let Some(libc_getauxval) = getauxval.get() { - unsafe { - let hwcap = libc_getauxval(c::AT_HWCAP) as usize; - let hwcap2 = libc_getauxval(c::AT_HWCAP2) as usize; - (hwcap, hwcap2) - } - } else { - (0, 0) + unsafe { + let hwcap = getauxval(c::AT_HWCAP) as usize; + let hwcap2 = getauxval(c::AT_HWCAP2) as usize; + (hwcap, hwcap2) } } @@ -61,9 +54,5 @@ pub(crate) fn linux_minsigstksz() -> usize { ))] #[inline] pub(crate) fn linux_execfn() -> &'static CStr { - if let Some(libc_getauxval) = getauxval.get() { - unsafe { CStr::from_ptr(libc_getauxval(c::AT_EXECFN).cast()) } - } else { - cstr!("") - } + unsafe { CStr::from_ptr(getauxval(c::AT_EXECFN).cast()) } } diff --git a/src/backend/linux_raw/arch/mod.rs b/src/backend/linux_raw/arch/mod.rs index ac9e25fa7..12c82be18 100644 --- a/src/backend/linux_raw/arch/mod.rs +++ b/src/backend/linux_raw/arch/mod.rs @@ -51,11 +51,11 @@ pub(in crate::backend) mod asm; ))] pub(in crate::backend) use self::asm as choose; -// On 32-bit x86, use vDSO wrappers for all syscalls. We could use the -// architecture syscall instruction (`int 0x80`), but the vDSO kernel_vsyscall +// On 32-bit x86, use the kernel_vsyscall mechanism for syscalls. We could use +// the architecture syscall instruction (`int 0x80`), but the kernel_vsyscall // mechanism is much faster. #[cfg(target_arch = "x86")] -pub(in crate::backend) use super::vdso_wrappers::x86_via_vdso as choose; +pub(in crate::backend) use super::x86_vsyscall as choose; // This would be the code for always using `int 0x80` on 32-bit x86. //#[cfg(target_arch = "x86")] diff --git a/src/backend/linux_raw/arch/x86.rs b/src/backend/linux_raw/arch/x86.rs index e789181cc..2429659e5 100644 --- a/src/backend/linux_raw/arch/x86.rs +++ b/src/backend/linux_raw/arch/x86.rs @@ -15,7 +15,7 @@ use crate::backend::reg::{ ArgReg, FromAsm, RetReg, SyscallNumber, ToAsm, A0, A1, A2, A3, A4, A5, R0, }; -use crate::backend::vdso_wrappers::SyscallType; +use crate::backend::x86_vsyscall::SyscallType; use core::arch::asm; #[inline] diff --git a/src/backend/linux_raw/mod.rs b/src/backend/linux_raw/mod.rs index 0d4e5332d..335507f91 100644 --- a/src/backend/linux_raw/mod.rs +++ b/src/backend/linux_raw/mod.rs @@ -18,10 +18,12 @@ mod arch; mod conv; mod reg; -#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))] +#[cfg(any(feature = "time", feature = "process"))] mod vdso; -#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))] +#[cfg(any(feature = "time", feature = "process"))] mod vdso_wrappers; +#[cfg(target_arch = "x86")] +mod x86_vsyscall; #[cfg(feature = "event")] pub(crate) mod event; diff --git a/src/backend/linux_raw/param/auxv.rs b/src/backend/linux_raw/param/auxv.rs index 59c7a17ad..92ff86c6c 100644 --- a/src/backend/linux_raw/param/auxv.rs +++ b/src/backend/linux_raw/param/auxv.rs @@ -21,6 +21,8 @@ use core::sync::atomic::AtomicU8; use core::sync::atomic::Ordering::Relaxed; use core::sync::atomic::{AtomicPtr, AtomicUsize}; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use linux_raw_sys::general::AT_SYSINFO; use linux_raw_sys::general::{ AT_BASE, AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR, @@ -38,8 +40,12 @@ pub(crate) fn page_size() -> usize { let mut page_size = PAGE_SIZE.load(Relaxed); if page_size == 0 { - init_auxv(); - page_size = PAGE_SIZE.load(Relaxed); + #[cold] + fn compute_page_size() -> usize { + init_auxv(); + PAGE_SIZE.load(Relaxed) + } + page_size = compute_page_size(); } page_size @@ -51,8 +57,12 @@ pub(crate) fn clock_ticks_per_second() -> u64 { let mut ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed); if ticks == 0 { - init_auxv(); - ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed); + #[cold] + fn compute_clock_ticks_per_second() -> usize { + init_auxv(); + CLOCK_TICKS_PER_SECOND.load(Relaxed) + } + ticks = compute_clock_ticks_per_second(); } ticks as u64 @@ -65,9 +75,12 @@ pub(crate) fn linux_hwcap() -> (usize, usize) { let mut hwcap2 = HWCAP2.load(Relaxed); if hwcap == 0 || hwcap2 == 0 { - init_auxv(); - hwcap = HWCAP.load(Relaxed); - hwcap2 = HWCAP2.load(Relaxed); + #[cold] + fn compute_linux_hwcap() -> (usize, usize) { + init_auxv(); + (HWCAP.load(Relaxed), HWCAP2.load(Relaxed)) + } + (hwcap, hwcap2) = compute_linux_hwcap(); } (hwcap, hwcap2) @@ -92,8 +105,12 @@ pub(crate) fn linux_execfn() -> &'static CStr { let mut execfn = EXECFN.load(Relaxed); if execfn.is_null() { - init_auxv(); - execfn = EXECFN.load(Relaxed); + #[cold] + fn compute_linux_execfn() -> *mut c::c_char { + init_auxv(); + EXECFN.load(Relaxed) + } + execfn = compute_linux_execfn(); } // SAFETY: We assume the `AT_EXECFN` value provided by the kernel is a @@ -108,8 +125,12 @@ pub(crate) fn linux_secure() -> bool { // 0 means not initialized yet. if secure == 0 { - init_auxv(); - secure = SECURE.load(Relaxed); + #[cold] + fn compute_linux_secure() -> u8 { + init_auxv(); + SECURE.load(Relaxed) + } + secure = compute_linux_secure(); } // 0 means not present. Libc `getauxval(AT_SECURE)` would return 0. @@ -125,11 +146,13 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) { let mut phent = PHENT.load(Relaxed); let mut phnum = PHNUM.load(Relaxed); - if phdr.is_null() || phnum == 0 { - init_auxv(); - phdr = PHDR.load(Relaxed); - phent = PHENT.load(Relaxed); - phnum = PHNUM.load(Relaxed); + if phdr.is_null() || phent == 0 || phnum == 0 { + #[cold] + fn compute_exe_phdrs() -> (*mut Elf_Phdr, usize, usize) { + init_auxv(); + (PHDR.load(Relaxed), PHENT.load(Relaxed), PHNUM.load(Relaxed)) + } + (phdr, phent, phnum) = compute_exe_phdrs(); } (phdr.cast(), phent, phnum) @@ -145,12 +168,16 @@ pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr { let mut ehdr = SYSINFO_EHDR.load(Relaxed); if ehdr.is_null() { - // Use `maybe_init_auxv` to to read the aux vectors if it can, but do - // nothing if it can't. If it can't, then we'll get a null pointer - // here, which our callers are prepared to deal with. - maybe_init_auxv(); + #[cold] + fn compute_sysinfo_ehdr() -> *mut Elf_Ehdr { + // Use `maybe_init_auxv` to to read the aux vectors if it can, but do + // nothing if it can't. If it can't, then we'll get a null pointer + // here, which our callers are prepared to deal with. + maybe_init_auxv(); + SYSINFO_EHDR.load(Relaxed) + } - ehdr = SYSINFO_EHDR.load(Relaxed); + ehdr = compute_sysinfo_ehdr(); } ehdr @@ -162,8 +189,12 @@ pub(crate) fn entry() -> usize { let mut entry = ENTRY.load(Relaxed); if entry == 0 { - init_auxv(); - entry = ENTRY.load(Relaxed); + #[cold] + fn compute_entry() -> usize { + init_auxv(); + ENTRY.load(Relaxed) + } + entry = compute_entry(); } entry @@ -175,13 +206,34 @@ pub(crate) fn random() -> *const [u8; 16] { let mut random = RANDOM.load(Relaxed); if random.is_null() { - init_auxv(); - random = RANDOM.load(Relaxed); + #[cold] + fn compute_random() -> *mut [u8; 16] { + init_auxv(); + RANDOM.load(Relaxed) + } + random = compute_random(); } random } +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c::c_void { + let mut vsyscall = VSYSCALL.load(Relaxed); + + if vsyscall.is_null() { + #[cold] + fn compute_vsyscall() -> *mut c::c_void { + init_auxv(); + VSYSCALL.load(Relaxed) + } + vsyscall = compute_vsyscall(); + } + + vsyscall +} + static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); static CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0); static HWCAP: AtomicUsize = AtomicUsize::new(0); @@ -201,6 +253,8 @@ static PHNUM: AtomicUsize = AtomicUsize::new(0); static ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(null_mut()); +#[cfg(target_arch = "x86")] +static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); const PR_GET_AUXV: c::c_int = 0x4155_5856; @@ -389,6 +443,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti let mut egid = None; #[cfg(feature = "runtime")] let mut random = null_mut(); + #[cfg(target_arch = "x86")] + let mut vsyscall = null_mut(); for Elf_auxv_t { a_type, a_val } in aux_iter { match a_type as _ { @@ -428,6 +484,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti AT_ENTRY => entry = a_val as usize, #[cfg(feature = "runtime")] AT_RANDOM => random = check_raw_pointer::<[u8; 16]>(a_val as *mut _)?.as_ptr(), + #[cfg(target_arch = "x86")] + AT_SYSINFO => vsyscall = a_val.cast(), AT_NULL => break, _ => (), @@ -464,6 +522,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti ENTRY.store(entry, Relaxed); #[cfg(feature = "runtime")] RANDOM.store(random, Relaxed); + #[cfg(target_arch = "x86")] + VSYSCALL.store(vsyscall, Relaxed); Some(()) } diff --git a/src/backend/linux_raw/param/init.rs b/src/backend/linux_raw/param/init.rs index 615f177a2..6da7cda2e 100644 --- a/src/backend/linux_raw/param/init.rs +++ b/src/backend/linux_raw/param/init.rs @@ -14,6 +14,8 @@ use core::ptr::{null_mut, read, NonNull}; use core::sync::atomic::AtomicBool; use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use linux_raw_sys::general::AT_SYSINFO; use linux_raw_sys::general::{ AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR, }; @@ -96,6 +98,12 @@ pub(crate) fn random() -> *const [u8; 16] { unsafe { RANDOM.load(Ordering::Relaxed) } } +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c_void { + unsafe { VSYSCALL.load(Ordering::Relaxed) } +} + static mut PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); static mut CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0); static mut HWCAP: AtomicUsize = AtomicUsize::new(0); @@ -118,6 +126,8 @@ static mut PHNUM: AtomicUsize = AtomicUsize::new(0); static mut ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static mut RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(NonNull::dangling().as_ptr()); +#[cfg(target_arch = "x86")] +static mut VSYSCALL: AtomicPtr = AtomicPtr::new(NonNull::dangling().as_ptr()); /// When "use-explicitly-provided-auxv" is enabled, we export a function to be /// called during initialization, and passed a pointer to the original @@ -170,6 +180,8 @@ unsafe fn init_from_auxp(mut auxp: *const Elf_auxv_t) { AT_ENTRY => ENTRY.store(a_val as usize, Ordering::Relaxed), #[cfg(feature = "runtime")] AT_RANDOM => RANDOM.store(a_val.cast::<[u8; 16]>(), Ordering::Relaxed), + #[cfg(feature = "x86")] + AT_SYSINFO => VSYSCALL.store(a_val.cast::(), Ordering::Relaxed), AT_NULL => break, _ => (), diff --git a/src/backend/linux_raw/param/libc_auxv.rs b/src/backend/linux_raw/param/libc_auxv.rs index c92e62a28..1b786541f 100644 --- a/src/backend/linux_raw/param/libc_auxv.rs +++ b/src/backend/linux_raw/param/libc_auxv.rs @@ -11,15 +11,12 @@ use crate::ffi::CStr; #[cfg(not(feature = "runtime"))] use core::ptr::null; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use { + core::ffi::c_void, core::ptr::null_mut, core::sync::atomic::AtomicPtr, + core::sync::atomic::Ordering::Relaxed, +}; -// `getauxval` wasn't supported in glibc until 2.16. Also this lets us use -// `*mut` as the return type to preserve strict provenance. -#[cfg(not(feature = "runtime"))] -weak!(fn getauxval(c::c_ulong) -> *mut c::c_void); - -// With the "runtime" feature, go ahead and depend on `getauxval` existing so -// that we never fail. -#[cfg(feature = "runtime")] extern "C" { fn getauxval(type_: c::c_ulong) -> *mut c::c_void; } @@ -38,6 +35,8 @@ const AT_RANDOM: c::c_ulong = 25; const AT_HWCAP2: c::c_ulong = 26; const AT_SECURE: c::c_ulong = 23; const AT_EXECFN: c::c_ulong = 31; +#[cfg(target_arch = "x86")] +const AT_SYSINFO: c::c_ulong = 32; const AT_SYSINFO_EHDR: c::c_ulong = 33; // Declare `sysconf` ourselves so that we don't depend on all of libc just for @@ -72,6 +71,9 @@ fn test_abi() { const_assert_eq!(self::AT_ENTRY, ::libc::AT_ENTRY); #[cfg(feature = "runtime")] const_assert_eq!(self::AT_RANDOM, ::libc::AT_RANDOM); + // TODO: Upstream x86's `AT_SYSINFO` to libc. + #[cfg(target_arch = "x86")] + const_assert_eq!(self::AT_SYSINFO, ::linux_raw_sys::general::AT_SYSINFO); } #[cfg(feature = "param")] @@ -89,18 +91,6 @@ pub(crate) fn clock_ticks_per_second() -> u64 { #[cfg(feature = "param")] #[inline] pub(crate) fn linux_hwcap() -> (usize, usize) { - #[cfg(not(feature = "runtime"))] - unsafe { - if let Some(libc_getauxval) = getauxval.get() { - let hwcap = libc_getauxval(AT_HWCAP) as usize; - let hwcap2 = libc_getauxval(AT_HWCAP2) as usize; - (hwcap, hwcap2) - } else { - (0, 0) - } - } - - #[cfg(feature = "runtime")] unsafe { let hwcap = getauxval(AT_HWCAP) as usize; let hwcap2 = getauxval(AT_HWCAP2) as usize; @@ -130,19 +120,7 @@ pub(crate) fn linux_minsigstksz() -> usize { #[cfg(feature = "param")] #[inline] pub(crate) fn linux_execfn() -> &'static CStr { - #[cfg(not(feature = "runtime"))] - unsafe { - if let Some(libc_getauxval) = getauxval.get() { - CStr::from_ptr(libc_getauxval(AT_EXECFN).cast()) - } else { - cstr!("") - } - } - - #[cfg(feature = "runtime")] - unsafe { - CStr::from_ptr(getauxval(AT_EXECFN).cast()) - } + unsafe { CStr::from_ptr(getauxval(AT_EXECFN).cast()) } } #[cfg(feature = "runtime")] @@ -166,19 +144,7 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) { /// if we don't see it, this function returns a null pointer. #[inline] pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr { - #[cfg(not(feature = "runtime"))] - unsafe { - if let Some(libc_getauxval) = getauxval.get() { - libc_getauxval(AT_SYSINFO_EHDR) as *const Elf_Ehdr - } else { - null() - } - } - - #[cfg(feature = "runtime")] - unsafe { - getauxval(AT_SYSINFO_EHDR) as *const Elf_Ehdr - } + unsafe { getauxval(AT_SYSINFO_EHDR) as *const Elf_Ehdr } } #[cfg(feature = "runtime")] @@ -192,3 +158,25 @@ pub(crate) fn entry() -> usize { pub(crate) fn random() -> *const [u8; 16] { unsafe { getauxval(AT_RANDOM) as *const [u8; 16] } } + +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c_void { + // We call this for every system call, so memoize the value. + static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); + + let mut vsyscall = VSYSCALL.load(Relaxed); + + if vsyscall.is_null() { + #[cold] + fn compute_vsyscall() -> *mut c_void { + let vsyscall = unsafe { getauxval(AT_SYSINFO) as *mut c_void }; + VSYSCALL.store(vsyscall, Relaxed); + vsyscall + } + + vsyscall = compute_vsyscall(); + } + + vsyscall +} diff --git a/src/backend/linux_raw/vdso_wrappers.rs b/src/backend/linux_raw/vdso_wrappers.rs index 441738f8d..8256b21ae 100644 --- a/src/backend/linux_raw/vdso_wrappers.rs +++ b/src/backend/linux_raw/vdso_wrappers.rs @@ -9,11 +9,7 @@ //! functions. #![allow(unsafe_code)] -#[cfg(target_arch = "x86")] -use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; use super::vdso; -#[cfg(target_arch = "x86")] -use core::arch::global_asm; #[cfg(feature = "process")] #[cfg(any( target_arch = "x86_64", @@ -135,130 +131,6 @@ pub(crate) fn sched_getcpu() -> usize { } } -#[cfg(target_arch = "x86")] -pub(super) mod x86_via_vdso { - use super::{transmute, ArgReg, Relaxed, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; - use crate::backend::arch::asm; - - #[inline] - pub(in crate::backend) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall0(callee, nr) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall1<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall1(callee, nr, a0) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall1_noreturn<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - ) -> ! { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall1_noreturn(callee, nr, a0) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall2<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall2(callee, nr, a0, a1) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall3<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall3(callee, nr, a0, a1, a2) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall4<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall4(callee, nr, a0, a1, a2, a3) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall5<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - a4: ArgReg<'a, A4>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall6<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - a4: ArgReg<'a, A4>, - a5: ArgReg<'a, A5>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5) - } - - // With the indirect call, it isn't meaningful to do a separate - // `_readonly` optimization. - #[allow(unused_imports)] - pub(in crate::backend) use { - syscall0 as syscall0_readonly, syscall1 as syscall1_readonly, - syscall2 as syscall2_readonly, syscall3 as syscall3_readonly, - syscall4 as syscall4_readonly, syscall5 as syscall5_readonly, - syscall6 as syscall6_readonly, - }; -} - #[cfg(feature = "time")] type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_int; @@ -271,12 +143,6 @@ type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_in ))] type GetcpuType = unsafe extern "C" fn(*mut u32, *mut u32, *mut c_void) -> c::c_int; -/// The underlying syscall functions are only called from asm, using the -/// special syscall calling convention to pass arguments and return values, -/// which the signature here doesn't reflect. -#[cfg(target_arch = "x86")] -pub(super) type SyscallType = unsafe extern "C" fn(); - /// Initialize `CLOCK_GETTIME` and return its value. #[cfg(feature = "time")] #[cold] @@ -303,16 +169,6 @@ fn init_getcpu() -> GetcpuType { unsafe { transmute(GETCPU.load(Relaxed)) } } -/// Initialize `SYSCALL` and return its value. -#[cfg(target_arch = "x86")] -#[cold] -fn init_syscall() -> SyscallType { - init(); - // SAFETY: Load the function address from static storage that we just - // initialized. - unsafe { transmute(SYSCALL.load(Relaxed)) } -} - /// `AtomicPtr` can't hold a `fn` pointer, so we use a `*` pointer to this /// placeholder type, and cast it as needed. struct Function; @@ -326,8 +182,6 @@ static mut CLOCK_GETTIME: AtomicPtr = AtomicPtr::new(null_mut()); target_arch = "powerpc64" ))] static mut GETCPU: AtomicPtr = AtomicPtr::new(null_mut()); -#[cfg(target_arch = "x86")] -static mut SYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); #[cfg(feature = "time")] unsafe extern "C" fn rustix_clock_gettime_via_syscall( @@ -405,34 +259,6 @@ unsafe extern "C" fn rustix_getcpu_via_syscall( } } -#[cfg(target_arch = "x86")] -extern "C" { - /// A symbol pointing to an `int 0x80` instruction. This “function” is only - /// called from assembly, and only with the x86 syscall calling convention, - /// so its signature here is not its true signature. - /// - /// This extern block and the `global_asm!` below can be replaced with - /// `#[naked]` if it's stabilized. - fn rustix_int_0x80(); -} - -#[cfg(target_arch = "x86")] -global_asm!( - r#" - .section .text.rustix_int_0x80,"ax",@progbits - .p2align 4 - .weak rustix_int_0x80 - .hidden rustix_int_0x80 - .type rustix_int_0x80, @function -rustix_int_0x80: - .cfi_startproc - int 0x80 - ret - .cfi_endproc - .size rustix_int_0x80, .-rustix_int_0x80 -"# -); - fn minimal_init() { // SAFETY: Store default function addresses in static storage so that if we // end up making any system calls while we read the vDSO, they'll work. If @@ -468,18 +294,6 @@ fn minimal_init() { ) .ok(); } - - #[cfg(target_arch = "x86")] - { - SYSCALL - .compare_exchange( - null_mut(), - rustix_int_0x80 as *mut Function, - Relaxed, - Relaxed, - ) - .ok(); - } } } @@ -591,18 +405,5 @@ fn init() { } } } - - // On x86, also look up the vsyscall entry point. - #[cfg(target_arch = "x86")] - { - let ptr = vdso.sym(cstr!("LINUX_2.5"), cstr!("__kernel_vsyscall")); - assert!(!ptr.is_null()); - - // SAFETY: As above, store the computed function addresses in - // static storage. - unsafe { - SYSCALL.store(ptr.cast(), Relaxed); - } - } } } diff --git a/src/backend/linux_raw/x86_vsyscall.rs b/src/backend/linux_raw/x86_vsyscall.rs new file mode 100644 index 000000000..b013fdaa8 --- /dev/null +++ b/src/backend/linux_raw/x86_vsyscall.rs @@ -0,0 +1,103 @@ +//! Implement syscalls using the x86 vsyscall mechanism. +//! +//! # Safety +//! +//! Similar to syscalls.rs, this file performs raw system calls, and sometimes +//! passes them uninitialized memory buffers. +#![allow(unsafe_code)] + +use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; +use crate::backend::arch::asm; +use core::mem::transmute; + +#[inline] +pub(super) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall0(callee, nr) +} + +#[inline] +pub(super) unsafe fn syscall1<'a>(nr: SyscallNumber<'a>, a0: ArgReg<'a, A0>) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall1(callee, nr, a0) +} + +#[inline] +pub(super) unsafe fn syscall1_noreturn<'a>(nr: SyscallNumber<'a>, a0: ArgReg<'a, A0>) -> ! { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall1_noreturn(callee, nr, a0) +} + +#[inline] +pub(super) unsafe fn syscall2<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall2(callee, nr, a0, a1) +} + +#[inline] +pub(super) unsafe fn syscall3<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall3(callee, nr, a0, a1, a2) +} + +#[inline] +pub(super) unsafe fn syscall4<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall4(callee, nr, a0, a1, a2, a3) +} + +#[inline] +pub(super) unsafe fn syscall5<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, + a4: ArgReg<'a, A4>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4) +} + +#[inline] +pub(super) unsafe fn syscall6<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, + a4: ArgReg<'a, A4>, + a5: ArgReg<'a, A5>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5) +} + +// With the indirect call, it isn't meaningful to do a separate +// `_readonly` optimization. +#[allow(unused_imports)] +pub(super) use { + syscall0 as syscall0_readonly, syscall1 as syscall1_readonly, syscall2 as syscall2_readonly, + syscall3 as syscall3_readonly, syscall4 as syscall4_readonly, syscall5 as syscall5_readonly, + syscall6 as syscall6_readonly, +}; + +/// The underlying syscall functions are only called from asm, using the +/// special syscall calling convention to pass arguments and return values, +/// which the signature here doesn't reflect. +pub(super) type SyscallType = unsafe extern "C" fn(); diff --git a/src/lib.rs b/src/lib.rs index 51fbdbaf7..623810f39 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -157,15 +157,9 @@ pub(crate) mod check_types; #[macro_use] pub(crate) mod bitcast; -// linux_raw: Weak symbols are used by the use-libc-auxv feature for -// glibc 2.15 support. -// -// libc: Weak symbols are used to call various functions available in some -// versions of libc and not others. -#[cfg(any( - all(linux_raw, feature = "use-libc-auxv"), - all(libc, not(any(windows, target_os = "espidf", target_os = "wasi"))) -))] +// Weak symbols are used to call various functions available in some versions +// of libc and not others. +#[cfg(all(libc, not(any(windows, target_os = "espidf", target_os = "wasi"))))] #[macro_use] mod weak; diff --git a/tests/param/auxv.rs b/tests/param/auxv.rs index 7806fa357..99e23860f 100644 --- a/tests/param/auxv.rs +++ b/tests/param/auxv.rs @@ -27,17 +27,17 @@ fn test_clock_ticks_per_second() { ))] #[test] fn test_linux_hwcap() { - weak!(fn getauxval(libc::c_ulong) -> libc::c_ulong); + extern "C" { + fn getauxval(type_: c::c_ulong) -> *mut c::c_void; + } - if let Some(libc_getauxval) = getauxval.get() { - let (_hwcap, hwcap2) = linux_hwcap(); + let (_hwcap, hwcap2) = linux_hwcap(); - // glibc seems to return a different value than `LD_SHOW_AUXV=1` reports. - #[cfg(not(target_env = "gnu"))] - assert_eq!(_hwcap, unsafe { libc_getauxval(libc::AT_HWCAP) } as usize); + // glibc seems to return a different value than `LD_SHOW_AUXV=1` reports. + #[cfg(not(target_env = "gnu"))] + assert_eq!(_hwcap, unsafe { getauxval(libc::AT_HWCAP) } as usize); - assert_eq!(hwcap2, unsafe { libc_getauxval(libc::AT_HWCAP2) } as usize); - } + assert_eq!(hwcap2, unsafe { getauxval(libc::AT_HWCAP2) } as usize); } #[cfg(any(