diff --git a/library/std/src/sys/thread_local/key/racy.rs b/library/std/src/sys/thread_local/key/racy.rs index a12ff7ac36ba5..0a10925ae6499 100644 --- a/library/std/src/sys/thread_local/key/racy.rs +++ b/library/std/src/sys/thread_local/key/racy.rs @@ -42,6 +42,7 @@ impl LazyKey { } } + #[cold] fn lazy_init(&self) -> usize { // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange // below relies on using KEY_SENTVAL as a sentinel value to check who won the diff --git a/library/std/src/sys/thread_local/key/xous.rs b/library/std/src/sys/thread_local/key/xous.rs index a27cec5ca1a60..f647bf59ff9e1 100644 --- a/library/std/src/sys/thread_local/key/xous.rs +++ b/library/std/src/sys/thread_local/key/xous.rs @@ -67,6 +67,7 @@ unsafe extern "Rust" { static DTORS: Atomic<*mut Node>; } +#[inline] fn tls_ptr_addr() -> *mut *mut u8 { let mut tp: usize; unsafe { @@ -80,14 +81,19 @@ fn tls_ptr_addr() -> *mut *mut u8 { /// Creates an area of memory that's unique per thread. This area will /// contain all thread local pointers. +#[inline] fn tls_table() -> &'static mut [*mut u8] { let tp = tls_ptr_addr(); if !tp.is_null() { - return unsafe { - core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>()) - }; + unsafe { core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>()) } + } else { + tls_table_slow() } +} + +#[cold] +fn tls_table_slow() -> &'static mut [*mut u8] { // If the TP register is `0`, then this thread hasn't initialized // its TLS yet. Allocate a new page to store this memory. let tp = unsafe { diff --git a/library/std/src/sys/thread_local/native/eager.rs b/library/std/src/sys/thread_local/native/eager.rs index fd48c4f720216..b91c072b5d892 100644 --- a/library/std/src/sys/thread_local/native/eager.rs +++ b/library/std/src/sys/thread_local/native/eager.rs @@ -4,7 +4,7 @@ use crate::sys::thread_local::{abort_on_dtor_unwind, destructors}; #[derive(Clone, Copy)] enum State { - Initial, + Uninitialized, Alive, Destroyed, } @@ -17,7 +17,7 @@ pub struct Storage { impl Storage { pub const fn new(val: T) -> Storage { - Storage { state: Cell::new(State::Initial), val: UnsafeCell::new(val) } + Storage { state: Cell::new(State::Uninitialized), val: UnsafeCell::new(val) } } /// Gets a pointer to the TLS value. If the TLS variable has been destroyed, @@ -30,16 +30,22 @@ impl Storage { /// The `self` reference must remain valid until the TLS destructor is run. #[inline] pub unsafe fn get(&self) -> *const T { - match self.state.get() { - State::Alive => self.val.get(), - State::Destroyed => ptr::null(), - State::Initial => unsafe { self.initialize() }, + if let State::Alive = self.state.get() { + self.val.get() + } else { + unsafe { self.get_or_init_slow() } } } #[cold] - unsafe fn initialize(&self) -> *const T { - // Register the destructor + unsafe fn get_or_init_slow(&self) -> *const T { + match self.state.get() { + State::Uninitialized => {} + State::Alive => return self.val.get(), + State::Destroyed => return ptr::null(), + } + + // Register the destructor. // SAFETY: // The caller guarantees that `self` will be valid until thread destruction. diff --git a/library/std/src/sys/thread_local/os.rs b/library/std/src/sys/thread_local/os.rs index fe6af27db3a17..a0c6d16a5d020 100644 --- a/library/std/src/sys/thread_local/os.rs +++ b/library/std/src/sys/thread_local/os.rs @@ -68,6 +68,7 @@ impl Storage { /// /// The resulting pointer may not be used after reentrant inialialization /// or thread destruction has occurred. + #[inline] pub fn get(&'static self, i: Option<&mut Option>, f: impl FnOnce() -> T) -> *const T { let key = self.key.force(); let ptr = unsafe { get(key) as *mut Value }; @@ -84,6 +85,7 @@ impl Storage { /// # Safety /// * `key` must be the result of calling `self.key.force()` /// * `ptr` must be the current value associated with `key`. + #[cold] unsafe fn try_initialize( key: Key, ptr: *mut Value,