linux-x86/1.75.0/lib/rustlib/src/rust/library/std/src/sys/common/thread_local/fast_local.rs - platform/prebuilts/rust - Git at Google

 use super::lazy::LazyKeyInner;
 use crate::cell::Cell;
 use crate::sys::thread_local_dtor::register_dtor;
 use crate::{fmt, mem, panic};

 #[doc(hidden)]
 #[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
 #[allow_internal_unsafe]
 #[unstable(feature = "thread_local_internals", issue = "none")]
 #[rustc_macro_transparency = "semitransparent"]
 pub macro thread_local_inner {
     // used to generate the `LocalKey` value for const-initialized thread locals
     (@key $t:ty, const $init:expr) => {{
         #[inline]
         #[deny(unsafe_op_in_unsafe_fn)]
         unsafe fn __getit(
             _init: $crate::option::Option<&mut $crate::option::Option<$t>>,
         ) -> $crate::option::Option<&'static $t> {
             const INIT_EXPR: $t = $init;
             // If the platform has support for `#[thread_local]`, use it.
             #[thread_local]
             static mut VAL: $t = INIT_EXPR;

             // If a dtor isn't needed we can do something "very raw" and
             // just get going.
             if !$crate::mem::needs_drop::<$t>() {
                 unsafe {
                     return $crate::option::Option::Some(&VAL)
                 }
             }

             // 0 == dtor not registered
             // 1 == dtor registered, dtor not run
             // 2 == dtor registered and is running or has run
             #[thread_local]
             static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0);

             // Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires
             // all that comes with it.
             unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
                 $crate::thread::local_impl::abort_on_dtor_unwind(|| {
                     let old_state = STATE.replace(2);
                     $crate::debug_assert_eq!(old_state, 1);
                     // Safety: safety requirement is passed on to caller.
                     unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); }
                 });
             }

             unsafe {
                 match STATE.get() {
                     // 0 == we haven't registered a destructor, so do
                     //   so now.
                     0 => {
                         $crate::thread::local_impl::Key::<$t>::register_dtor(
                             $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
                             destroy,
                         );
                         STATE.set(1);
                         $crate::option::Option::Some(&VAL)
                     }
                     // 1 == the destructor is registered and the value
                     //   is valid, so return the pointer.
                     1 => $crate::option::Option::Some(&VAL),
                     // otherwise the destructor has already run, so we
                     // can't give access.
                     _ => $crate::option::Option::None,
                 }
             }
         }

         unsafe {
             $crate::thread::LocalKey::new(__getit)
         }
     }},

     // used to generate the `LocalKey` value for `thread_local!`
     (@key $t:ty, $init:expr) => {
         {
             #[inline]
             fn __init() -> $t { $init }

             #[inline]
             unsafe fn __getit(
                 init: $crate::option::Option<&mut $crate::option::Option<$t>>,
             ) -> $crate::option::Option<&'static $t> {
                 #[thread_local]
                 static __KEY: $crate::thread::local_impl::Key<$t> =
                     $crate::thread::local_impl::Key::<$t>::new();

                 unsafe {
                     __KEY.get(move || {
                         if let $crate::option::Option::Some(init) = init {
                             if let $crate::option::Option::Some(value) = init.take() {
                                 return value;
                             } else if $crate::cfg!(debug_assertions) {
                                 $crate::unreachable!("missing default value");
                             }
                         }
                         __init()
                     })
                 }
             }

             unsafe {
                 $crate::thread::LocalKey::new(__getit)
             }
         }
     },
     ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
         $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
             $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
     },
 }

 #[derive(Copy, Clone)]
 enum DtorState {
     Unregistered,
     Registered,
     RunningOrHasRun,
 }

 // This data structure has been carefully constructed so that the fast path
 // only contains one branch on x86. That optimization is necessary to avoid
 // duplicated tls lookups on OSX.
 //
 // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
 pub struct Key<T> {
     // If `LazyKeyInner::get` returns `None`, that indicates either:
     //   * The value has never been initialized
     //   * The value is being recursively initialized
     //   * The value has already been destroyed or is being destroyed
     // To determine which kind of `None`, check `dtor_state`.
     //
     // This is very optimizer friendly for the fast path - initialized but
     // not yet dropped.
     inner: LazyKeyInner<T>,

     // Metadata to keep track of the state of the destructor. Remember that
     // this variable is thread-local, not global.
     dtor_state: Cell<DtorState>,
 }

 impl<T> fmt::Debug for Key<T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("Key").finish_non_exhaustive()
     }
 }
 impl<T> Key<T> {
     pub const fn new() -> Key<T> {
         Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
     }

     // note that this is just a publicly-callable function only for the
     // const-initialized form of thread locals, basically a way to call the
     // free `register_dtor` function defined elsewhere in std.
     pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
         unsafe {
             register_dtor(a, dtor);
         }
     }

     pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
         // SAFETY: See the definitions of `LazyKeyInner::get` and
         // `try_initialize` for more information.
         //
         // The caller must ensure no mutable references are ever active to
         // the inner cell or the inner T when this is called.
         // The `try_initialize` is dependant on the passed `init` function
         // for this.
         unsafe {
             match self.inner.get() {
                 Some(val) => Some(val),
                 None => self.try_initialize(init),
             }
         }
     }

     // `try_initialize` is only called once per fast thread local variable,
     // except in corner cases where thread_local dtors reference other
     // thread_local's, or it is being recursively initialized.
     //
     // Macos: Inlining this function can cause two `tlv_get_addr` calls to
     // be performed for every call to `Key::get`.
     // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
     #[inline(never)]
     unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
         // SAFETY: See comment above (this function doc).
         if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
             // SAFETY: See comment above (this function doc).
             Some(unsafe { self.inner.initialize(init) })
         } else {
             None
         }
     }

     // `try_register_dtor` is only called once per fast thread local
     // variable, except in corner cases where thread_local dtors reference
     // other thread_local's, or it is being recursively initialized.
     unsafe fn try_register_dtor(&self) -> bool {
         match self.dtor_state.get() {
             DtorState::Unregistered => {
                 // SAFETY: dtor registration happens before initialization.
                 // Passing `self` as a pointer while using `destroy_value<T>`
                 // is safe because the function will build a pointer to a
                 // Key<T>, which is the type of self and so find the correct
                 // size.
                 unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
                 self.dtor_state.set(DtorState::Registered);
                 true
             }
             DtorState::Registered => {
                 // recursively initialized
                 true
             }
             DtorState::RunningOrHasRun => false,
         }
     }
 }

 unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
     let ptr = ptr as *mut Key<T>;

     // SAFETY:
     //
     // The pointer `ptr` has been built just above and comes from
     // `try_register_dtor` where it is originally a Key<T> coming from `self`,
     // making it non-NUL and of the correct type.
     //
     // Right before we run the user destructor be sure to set the
     // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
     // causes future calls to `get` to run `try_initialize_drop` again,
     // which will now fail, and return `None`.
     //
     // Wrap the call in a catch to ensure unwinding is caught in the event
     // a panic takes place in a destructor.
     if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
         let value = (*ptr).inner.take();
         (*ptr).dtor_state.set(DtorState::RunningOrHasRun);
         drop(value);
     })) {
         rtabort!("thread local panicked on drop");
     }
 }
	use super::lazy::LazyKeyInner;
	use crate::cell::Cell;
	use crate::sys::thread_local_dtor::register_dtor;
	use crate::{fmt, mem, panic};

	#[doc(hidden)]
	#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
	#[allow_internal_unsafe]
	#[unstable(feature = "thread_local_internals", issue = "none")]
	#[rustc_macro_transparency = "semitransparent"]
	pub macro thread_local_inner {
	// used to generate the `LocalKey` value for const-initialized thread locals
	(@key $t:ty, const $init:expr) => {{
	#[inline]
	#[deny(unsafe_op_in_unsafe_fn)]
	unsafe fn __getit(
	_init: $crate::option::Option<&mut $crate::option::Option<$t>>,
	) -> $crate::option::Option<&'static $t> {
	const INIT_EXPR: $t = $init;
	// If the platform has support for `#[thread_local]`, use it.
	#[thread_local]
	static mut VAL: $t = INIT_EXPR;

	// If a dtor isn't needed we can do something "very raw" and
	// just get going.
	if !$crate::mem::needs_drop::<$t>() {
	unsafe {
	return $crate::option::Option::Some(&VAL)
	}
	}

	// 0 == dtor not registered
	// 1 == dtor registered, dtor not run
	// 2 == dtor registered and is running or has run
	#[thread_local]
	static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0);

	// Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires
	// all that comes with it.
	unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
	$crate::thread::local_impl::abort_on_dtor_unwind(\|\| {
	let old_state = STATE.replace(2);
	$crate::debug_assert_eq!(old_state, 1);
	// Safety: safety requirement is passed on to caller.
	unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); }
	});
	}

	unsafe {
	match STATE.get() {
	// 0 == we haven't registered a destructor, so do
	// so now.
	0 => {
	$crate::thread::local_impl::Key::<$t>::register_dtor(
	$crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
	destroy,
	);
	STATE.set(1);
	$crate::option::Option::Some(&VAL)
	}
	// 1 == the destructor is registered and the value
	// is valid, so return the pointer.
	1 => $crate::option::Option::Some(&VAL),
	// otherwise the destructor has already run, so we
	// can't give access.
	_ => $crate::option::Option::None,
	}
	}
	}

	unsafe {
	$crate::thread::LocalKey::new(__getit)
	}
	}},

	// used to generate the `LocalKey` value for `thread_local!`
	(@key $t:ty, $init:expr) => {
	{
	#[inline]
	fn __init() -> $t { $init }

	#[inline]
	unsafe fn __getit(
	init: $crate::option::Option<&mut $crate::option::Option<$t>>,
	) -> $crate::option::Option<&'static $t> {
	#[thread_local]
	static __KEY: $crate::thread::local_impl::Key<$t> =
	$crate::thread::local_impl::Key::<$t>::new();

	unsafe {
	__KEY.get(move \|\| {
	if let $crate::option::Option::Some(init) = init {
	if let $crate::option::Option::Some(value) = init.take() {
	return value;
	} else if $crate::cfg!(debug_assertions) {
	$crate::unreachable!("missing default value");
	}
	}
	__init()
	})
	}
	}

	unsafe {
	$crate::thread::LocalKey::new(__getit)
	}
	}
	},
	($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
	$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
	$crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
	},
	}

	#[derive(Copy, Clone)]
	enum DtorState {
	Unregistered,
	Registered,
	RunningOrHasRun,
	}

	// This data structure has been carefully constructed so that the fast path
	// only contains one branch on x86. That optimization is necessary to avoid
	// duplicated tls lookups on OSX.
	//
	// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
	pub struct Key<T> {
	// If `LazyKeyInner::get` returns `None`, that indicates either:
	// * The value has never been initialized
	// * The value is being recursively initialized
	// * The value has already been destroyed or is being destroyed
	// To determine which kind of `None`, check `dtor_state`.
	//
	// This is very optimizer friendly for the fast path - initialized but
	// not yet dropped.
	inner: LazyKeyInner<T>,

	// Metadata to keep track of the state of the destructor. Remember that
	// this variable is thread-local, not global.
	dtor_state: Cell<DtorState>,
	}

	impl<T> fmt::Debug for Key<T> {
	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
	f.debug_struct("Key").finish_non_exhaustive()
	}
	}
	impl<T> Key<T> {
	pub const fn new() -> Key<T> {
	Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
	}

	// note that this is just a publicly-callable function only for the
	// const-initialized form of thread locals, basically a way to call the
	// free `register_dtor` function defined elsewhere in std.
	pub unsafe fn register_dtor(a: mut u8, dtor: unsafe extern "C" fn(mut u8)) {
	unsafe {
	register_dtor(a, dtor);
	}
	}

	pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
	// SAFETY: See the definitions of `LazyKeyInner::get` and
	// `try_initialize` for more information.
	//
	// The caller must ensure no mutable references are ever active to
	// the inner cell or the inner T when this is called.
	// The `try_initialize` is dependant on the passed `init` function
	// for this.
	unsafe {
	match self.inner.get() {
	Some(val) => Some(val),
	None => self.try_initialize(init),
	}
	}
	}

	// `try_initialize` is only called once per fast thread local variable,
	// except in corner cases where thread_local dtors reference other
	// thread_local's, or it is being recursively initialized.
	//
	// Macos: Inlining this function can cause two `tlv_get_addr` calls to
	// be performed for every call to `Key::get`.
	// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
	#[inline(never)]
	unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
	// SAFETY: See comment above (this function doc).
	if !mem::needs_drop::<T>() \|\| unsafe { self.try_register_dtor() } {
	// SAFETY: See comment above (this function doc).
	Some(unsafe { self.inner.initialize(init) })
	} else {
	None
	}
	}

	// `try_register_dtor` is only called once per fast thread local
	// variable, except in corner cases where thread_local dtors reference
	// other thread_local's, or it is being recursively initialized.
	unsafe fn try_register_dtor(&self) -> bool {
	match self.dtor_state.get() {
	DtorState::Unregistered => {
	// SAFETY: dtor registration happens before initialization.
	// Passing `self` as a pointer while using `destroy_value<T>`
	// is safe because the function will build a pointer to a
	// Key<T>, which is the type of self and so find the correct
	// size.
	unsafe { register_dtor(self as const _ as mut u8, destroy_value::<T>) };
	self.dtor_state.set(DtorState::Registered);
	true
	}
	DtorState::Registered => {
	// recursively initialized
	true
	}
	DtorState::RunningOrHasRun => false,
	}
	}
	}

	unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
	let ptr = ptr as *mut Key<T>;

	// SAFETY:
	//
	// The pointer `ptr` has been built just above and comes from
	// `try_register_dtor` where it is originally a Key<T> coming from `self`,
	// making it non-NUL and of the correct type.
	//
	// Right before we run the user destructor be sure to set the
	// `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
	// causes future calls to `get` to run `try_initialize_drop` again,
	// which will now fail, and return `None`.
	//
	// Wrap the call in a catch to ensure unwinding is caught in the event
	// a panic takes place in a destructor.
	if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(\|\| unsafe {
	let value = (*ptr).inner.take();
	(*ptr).dtor_state.set(DtorState::RunningOrHasRun);
	drop(value);
	})) {
	rtabort!("thread local panicked on drop");
	}
	}