diff --git a/src/backend/linux_raw/process/syscalls.rs b/src/backend/linux_raw/process/syscalls.rs index 85c6fbbad..4eb9e0825 100644 --- a/src/backend/linux_raw/process/syscalls.rs +++ b/src/backend/linux_raw/process/syscalls.rs @@ -41,7 +41,8 @@ use {crate::backend::conv::slice_just_addr_mut, crate::process::Gid}; target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x" ))] pub(crate) use crate::backend::vdso_wrappers::sched_getcpu; @@ -50,7 +51,8 @@ pub(crate) use crate::backend::vdso_wrappers::sched_getcpu; target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x" )))] #[inline] pub(crate) fn sched_getcpu() -> usize { diff --git a/src/backend/linux_raw/vdso.rs b/src/backend/linux_raw/vdso.rs index 473cdee21..3179d1203 100644 --- a/src/backend/linux_raw/vdso.rs +++ b/src/backend/linux_raw/vdso.rs @@ -5,6 +5,10 @@ //! with Creative Commons Zero License, version 1.0, //! available at //! +//! It also incorporates the patch at: +//! , +//! with changes to fix the pointer arithmetic on s390x. +//! //! # Safety //! //! Parsing the vDSO involves a lot of raw pointer manipulation. This @@ -34,6 +38,7 @@ pub(super) struct Vdso { // Symbol table symtab: *const Elf_Sym, symstrings: *const u8, + gnu_hash: *const u32, bucket: *const ElfHashEntry, chain: *const ElfHashEntry, nbucket: ElfHashEntry, @@ -60,6 +65,16 @@ fn elf_hash(name: &CStr) -> u32 { h } +fn gnu_hash(name: &CStr) -> u32 { + let mut h: u32 = 5381; + for s in name.to_bytes() { + h = h + .wrapping_add(h.wrapping_mul(32)) + .wrapping_add(u32::from(*s)); + } + h +} + /// Create a `Vdso` value by parsing the vDSO at the `sysinfo_ehdr` address. fn init_from_sysinfo_ehdr() -> Option { // SAFETY: The auxv initialization code does extensive checks to ensure @@ -80,6 +95,7 @@ fn init_from_sysinfo_ehdr() -> Option { pv_offset: 0, symtab: null(), symstrings: null(), + gnu_hash: null(), bucket: null(), chain: null(), nbucket: 0, @@ -159,6 +175,11 @@ fn init_from_sysinfo_ehdr() -> Option { )? .as_ptr(); } + DT_GNU_HASH => { + vdso.gnu_hash = + check_raw_pointer::(vdso.addr_from_elf(d.d_un.d_ptr)? as *mut _)? + .as_ptr() + } DT_VERSYM => { vdso.versym = check_raw_pointer::(vdso.addr_from_elf(d.d_un.d_ptr)? as *mut _)? @@ -183,7 +204,10 @@ fn init_from_sysinfo_ehdr() -> Option { // `check_raw_pointer` will have checked these pointers for null, // however they could still be null if the expected dynamic table // entries are absent. - if vdso.symstrings.is_null() || vdso.symtab.is_null() || hash.is_null() { + if vdso.symstrings.is_null() + || vdso.symtab.is_null() + || (hash.is_null() && vdso.gnu_hash.is_null()) + { return None; // Failed } @@ -192,10 +216,21 @@ fn init_from_sysinfo_ehdr() -> Option { } // Parse the hash table header. - vdso.nbucket = *hash.add(0); - //vdso.nchain = *hash.add(1); - vdso.bucket = hash.add(2); - vdso.chain = hash.add(vdso.nbucket as usize + 2); + if !vdso.gnu_hash.is_null() { + vdso.nbucket = ElfHashEntry::from(*vdso.gnu_hash); + // The bucket array is located after the header (4 uint32) and the bloom + // filter (size_t array of gnu_hash[2] elements). + vdso.bucket = vdso + .gnu_hash + .add(4) + .add(size_of::() / 4 * *vdso.gnu_hash.add(2) as usize) + .cast(); + } else { + vdso.nbucket = *hash.add(0); + //vdso.nchain = *hash.add(1); + vdso.bucket = hash.add(2); + vdso.chain = hash.add(vdso.nbucket as usize + 2); + } // That's all we need. Some(vdso) @@ -261,49 +296,110 @@ impl Vdso { && (name == CStr::from_ptr(self.symstrings.add(aux.vda_name as usize).cast())) } + /// Check to see if the symbol is the one we're looking for. + /// + /// # Safety + /// + /// The raw pointers inside `self` must be valid. + unsafe fn check_sym( + &self, + sym: &Elf_Sym, + i: ElfHashEntry, + name: &CStr, + version: &CStr, + ver_hash: u32, + ) -> bool { + // Check for a defined global or weak function w/ right name. + // + // Accept `STT_NOTYPE` in addition to `STT_FUNC` for the symbol + // type, for compatibility with some versions of Linux on + // PowerPC64. See [this commit] in Linux for more background. + // + // [this commit]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/tools/testing/selftests/vDSO/parse_vdso.c?id=0161bd38c24312853ed5ae9a425a1c41c4ac674a + if ELF_ST_TYPE(sym.st_info) != STT_FUNC && ELF_ST_TYPE(sym.st_info) != STT_NOTYPE { + return false; + } + if ELF_ST_BIND(sym.st_info) != STB_GLOBAL && ELF_ST_BIND(sym.st_info) != STB_WEAK { + return false; + } + if name != CStr::from_ptr(self.symstrings.add(sym.st_name as usize).cast()) { + return false; + } + + // Check symbol version. + if !self.versym.is_null() + && !self.match_version(*self.versym.add(i as usize), version, ver_hash) + { + return false; + } + + true + } + /// Look up a symbol in the vDSO. pub(super) fn sym(&self, version: &CStr, name: &CStr) -> *mut c::c_void { let ver_hash = elf_hash(version); - let name_hash = elf_hash(name); // SAFETY: The pointers in `self` must be valid. unsafe { - let mut chain = *self - .bucket - .add((ElfHashEntry::from(name_hash) % self.nbucket) as usize); - - while chain != ElfHashEntry::from(STN_UNDEF) { - let sym = &*self.symtab.add(chain as usize); - - // Check for a defined global or weak function w/ right name. - // - // Accept `STT_NOTYPE` in addition to `STT_FUNC` for the symbol - // type, for compatibility with some versions of Linux on - // PowerPC64. See [this commit] in Linux for more background. - // - // [this commit]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/tools/testing/selftests/vDSO/parse_vdso.c?id=0161bd38c24312853ed5ae9a425a1c41c4ac674a - if (ELF_ST_TYPE(sym.st_info) != STT_FUNC && - ELF_ST_TYPE(sym.st_info) != STT_NOTYPE) - || (ELF_ST_BIND(sym.st_info) != STB_GLOBAL - && ELF_ST_BIND(sym.st_info) != STB_WEAK) - || sym.st_shndx == SHN_UNDEF - || sym.st_shndx == SHN_ABS - || ELF_ST_VISIBILITY(sym.st_other) != STV_DEFAULT - || (name != CStr::from_ptr(self.symstrings.add(sym.st_name as usize).cast())) - // Check symbol version. - || (!self.versym.is_null() - && !self.match_version(*self.versym.add(chain as usize), version, ver_hash)) - { - chain = *self.chain.add(chain as usize); - continue; + if !self.gnu_hash.is_null() { + let mut h1: u32 = gnu_hash(name); + + // Changes to fix the pointer arithmetic on s390x: cast + // `self.bucket` to `*const u32` here, because even though + // s390x's `ElfHashEntry` is 64-bit for `DT_HASH` tables, + // it uses 32-bit entries for `DT_GNU_HASH` tables. + let mut i = *self + .bucket + .cast::() + .add((ElfHashEntry::from(h1) % self.nbucket) as usize); + if i == 0 { + return null_mut(); + } + h1 |= 1; + // Changes to fix the pointer arithmetic on s390x: As above, + // cast `self.bucket` to `*const u32`. + let mut hashval = self + .bucket + .cast::() + .add(self.nbucket as usize) + .add((i - *self.gnu_hash.add(1)) as usize); + loop { + let sym: &Elf_Sym = &*self.symtab.add(i as usize); + let h2 = *hashval; + hashval = hashval.add(1); + if h1 == (h2 | 1) + && self.check_sym(sym, ElfHashEntry::from(i), name, version, ver_hash) + { + let sum = self.addr_from_elf(sym.st_value).unwrap(); + assert!( + sum as usize >= self.load_addr as usize + && sum as usize <= self.load_end as usize + ); + return sum as *mut c::c_void; + } + if (h2 & 1) != 0 { + break; + } + i += 1; + } + } else { + let mut i = *self + .bucket + .add((ElfHashEntry::from(elf_hash(name)) % self.nbucket) as usize); + while i != 0 { + let sym: &Elf_Sym = &*self.symtab.add(i as usize); + if sym.st_shndx != SHN_UNDEF && self.check_sym(sym, i, name, version, ver_hash) + { + let sum = self.addr_from_elf(sym.st_value).unwrap(); + assert!( + sum as usize >= self.load_addr as usize + && sum as usize <= self.load_end as usize + ); + return sum as *mut c::c_void; + } + i = *self.chain.add(i as usize); } - - let sum = self.addr_from_elf(sym.st_value).unwrap(); - assert!( - sum as usize >= self.load_addr as usize - && sum as usize <= self.load_end as usize - ); - return sum as *mut c::c_void; } } @@ -324,32 +420,103 @@ impl Vdso { } } +// Disable on MIPS since QEMU on MIPS doesn't provide a vDSO. #[cfg(linux_raw)] #[test] -#[ignore] // Until rustix is updated to the new vDSO format. +#[cfg_attr(any(target_arch = "mips", target_arch = "mips64"), ignore)] fn test_vdso() { let vdso = Vdso::new().unwrap(); assert!(!vdso.symtab.is_null()); assert!(!vdso.symstrings.is_null()); - #[cfg(target_arch = "x86_64")] - let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime")); - #[cfg(target_arch = "arm")] - let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64")); - #[cfg(target_arch = "aarch64")] - let ptr = vdso.sym(cstr!("LINUX_2.6.39"), cstr!("__kernel_clock_gettime")); - #[cfg(target_arch = "x86")] - let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64")); - #[cfg(target_arch = "riscv64")] - let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_clock_gettime")); - #[cfg(target_arch = "powerpc64")] - let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_clock_gettime")); - #[cfg(target_arch = "s390x")] - let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_clock_gettime")); - #[cfg(any(target_arch = "mips", target_arch = "mips32r6"))] - let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64")); - #[cfg(any(target_arch = "mips64", target_arch = "mips64r6"))] - let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime")); - - assert!(!ptr.is_null()); + { + #[cfg(target_arch = "x86_64")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime")); + #[cfg(target_arch = "arm")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64")); + #[cfg(target_arch = "aarch64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.39"), cstr!("__kernel_clock_gettime")); + #[cfg(target_arch = "x86")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64")); + #[cfg(target_arch = "riscv64")] + let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_clock_gettime")); + #[cfg(target_arch = "powerpc64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_clock_gettime")); + #[cfg(target_arch = "s390x")] + let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_clock_gettime")); + #[cfg(any(target_arch = "mips", target_arch = "mips32r6"))] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64")); + #[cfg(any(target_arch = "mips64", target_arch = "mips64r6"))] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime")); + + assert!(!ptr.is_null()); + } + + { + #[cfg(target_arch = "x86_64")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_getres")); + #[cfg(target_arch = "arm")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_getres")); + #[cfg(target_arch = "aarch64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.39"), cstr!("__kernel_clock_getres")); + #[cfg(target_arch = "x86")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_getres")); + #[cfg(target_arch = "riscv64")] + let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_clock_getres")); + #[cfg(target_arch = "powerpc64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_clock_getres")); + #[cfg(target_arch = "s390x")] + let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_clock_getres")); + #[cfg(any(target_arch = "mips", target_arch = "mips32r6"))] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_getres")); + #[cfg(any(target_arch = "mips64", target_arch = "mips64r6"))] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_getres")); + + assert!(!ptr.is_null()); + } + + { + #[cfg(target_arch = "x86_64")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_gettimeofday")); + #[cfg(target_arch = "arm")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_gettimeofday")); + #[cfg(target_arch = "aarch64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.39"), cstr!("__kernel_gettimeofday")); + #[cfg(target_arch = "x86")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_gettimeofday")); + #[cfg(target_arch = "riscv64")] + let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_gettimeofday")); + #[cfg(target_arch = "powerpc64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_gettimeofday")); + #[cfg(target_arch = "s390x")] + let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_gettimeofday")); + #[cfg(any(target_arch = "mips", target_arch = "mips32r6"))] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_gettimeofday")); + #[cfg(any(target_arch = "mips64", target_arch = "mips64r6"))] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_gettimeofday")); + + assert!(!ptr.is_null()); + } + + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "riscv64", + target_arch = "powerpc64", + target_arch = "s390x", + ))] + { + #[cfg(target_arch = "x86_64")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_getcpu")); + #[cfg(target_arch = "x86")] + let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_getcpu")); + #[cfg(target_arch = "riscv64")] + let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_getcpu")); + #[cfg(target_arch = "powerpc64")] + let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_getcpu")); + #[cfg(target_arch = "s390x")] + let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_getcpu")); + + assert!(!ptr.is_null()); + } } diff --git a/src/backend/linux_raw/vdso_wrappers.rs b/src/backend/linux_raw/vdso_wrappers.rs index ce1facfaf..d32054f3d 100644 --- a/src/backend/linux_raw/vdso_wrappers.rs +++ b/src/backend/linux_raw/vdso_wrappers.rs @@ -20,7 +20,8 @@ use core::arch::global_asm; target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] use core::ffi::c_void; use core::mem::transmute; @@ -37,7 +38,8 @@ use linux_raw_sys::general::timespec as __kernel_old_timespec; target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x" ) ), feature = "time" @@ -117,7 +119,8 @@ pub(crate) fn clock_gettime_dynamic(which_clock: DynamicClockId<'_>) -> io::Resu target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] #[inline] pub(crate) fn sched_getcpu() -> usize { @@ -268,7 +271,8 @@ type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_in target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] type GetcpuType = unsafe extern "C" fn(*mut u32, *mut u32, *mut c_void) -> c::c_int; @@ -294,7 +298,8 @@ fn init_clock_gettime() -> ClockGettimeType { target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] #[cold] fn init_getcpu() -> GetcpuType { @@ -324,7 +329,8 @@ static CLOCK_GETTIME: AtomicPtr = AtomicPtr::new(null_mut()); target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] static GETCPU: AtomicPtr = AtomicPtr::new(null_mut()); #[cfg(target_arch = "x86")] @@ -393,7 +399,8 @@ unsafe fn _rustix_clock_gettime_via_syscall( target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] unsafe extern "C" fn rustix_getcpu_via_syscall( cpu: *mut u32, @@ -456,7 +463,8 @@ fn minimal_init() { target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] { GETCPU @@ -542,7 +550,8 @@ fn init() { target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x", ))] { // Look up the platform-specific `getcpu` symbol as documented @@ -557,11 +566,14 @@ fn init() { let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_getcpu")); #[cfg(target_arch = "powerpc64")] let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_getcpu")); + #[cfg(target_arch = "s390x")] + let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_getcpu")); #[cfg(any( target_arch = "x86_64", target_arch = "riscv64", - target_arch = "powerpc64" + target_arch = "powerpc64", + target_arch = "s390x" ))] let ok = true; @@ -576,7 +588,6 @@ fn init() { target_arch = "mips32r6", target_arch = "mips64", target_arch = "mips64r6", - target_arch = "s390x", ))] let ok = false;