cache.rs - source
Description: Source of the Rust file `library/std_detect/src/detect/cache.rs`.

#
std_detect/detect/
cache.rs

    1//! Caches run-time feature detection so that it only needs to be computed
    2//! once.
    3
    4#![allow(dead_code)] // not used on all platforms
    5
    6use core::sync::atomic::{AtomicUsize, Ordering};
    7
    8/// Sets the `bit` of `x`.
    9#[inline]
    10const fn set_bit(x: u128, bit: u32) -> u128 {
    11    x | 1 << bit
    12}
    13
    14/// Tests the `bit` of `x`.
    15#[inline]
    16const fn test_bit(x: u128, bit: u32) -> bool {
    17    x & (1 << bit) != 0
    18}
    19
    20/// Unset the `bit of `x`.
    21#[inline]
    22const fn unset_bit(x: u128, bit: u32) -> u128 {
    23    x & !(1 << bit)
    24}
    25
    26/// Maximum number of features that can be cached.
    27const CACHE_CAPACITY: u32 = 93;
    28
    29/// This type is used to initialize the cache
    30// The derived `Default` implementation will initialize the field to zero,
    31// which is what we want.
    32#[derive(Copy, Clone, Default, PartialEq, Eq)]
    33pub(crate) struct Initializer(u128);
    34
    35// NOTE: the `debug_assert!` would catch that we do not add more Features than
    36// the one fitting our cache.
    37impl Initializer {
    38    /// Tests the `bit` of the cache.
    39    #[inline]
    40    pub(crate) fn test(self, bit: u32) -> bool {
    41        debug_assert!(bit < CACHE_CAPACITY, "too many features, time to increase the cache size!");
    42        test_bit(self.0, bit)
    43    }
    44
    45    /// Sets the `bit` of the cache.
    46    #[inline]
    47    pub(crate) fn set(&mut self, bit: u32) {
    48        debug_assert!(bit < CACHE_CAPACITY, "too many features, time to increase the cache size!");
    49        let v = self.0;
    50        self.0 = set_bit(v, bit);
    51    }
    52
    53    /// Unsets the `bit` of the cache.
    54    #[inline]
    55    pub(crate) fn unset(&mut self, bit: u32) {
    56        debug_assert!(bit < CACHE_CAPACITY, "too many features, time to increase the cache size!");
    57        let v = self.0;
    58        self.0 = unset_bit(v, bit);
    59    }
    60}
    61
    62/// This global variable is a cache of the features supported by the CPU.
    63// Note: the third slot is only used in x86
    64// Another Slot can be added if needed without any change to `Initializer`
    65static CACHE: [Cache; 3] = [Cache::uninitialized(), Cache::uninitialized(), Cache::uninitialized()];
    66
    67/// Feature cache with capacity for `size_of::<usize>() * 8 - 1` features.
    68///
    69/// Note: 0 is used to represent an uninitialized cache, and (at least) the most
    70/// significant bit is set on any cache which has been initialized.
    71///
    72/// Note: we use `Relaxed` atomic operations, because we are only interested in
    73/// the effects of operations on a single memory location. That is, we only need
    74/// "modification order", and not the full-blown "happens before".
    75struct Cache(AtomicUsize);
    76
    77impl Cache {
    78    const CAPACITY: u32 = (core::mem::size_of::<usize>() * 8 - 1) as u32;
    79    const MASK: usize = (1 << Cache::CAPACITY) - 1;
    80    const INITIALIZED_BIT: usize = 1usize << Cache::CAPACITY;
    81
    82    /// Creates an uninitialized cache.
    83    #[allow(clippy::declare_interior_mutable_const)]
    84    const fn uninitialized() -> Self {
    85        Cache(AtomicUsize::new(0))
    86    }
    87
    88    /// Is the `bit` in the cache set? Returns `None` if the cache has not been initialized.
    89    #[inline]
    90    pub(crate) fn test(&self, bit: u32) -> Option<bool> {
    91        let cached = self.0.load(Ordering::Relaxed);
    92        if cached == 0 { None } else { Some(test_bit(cached as u128, bit)) }
    93    }
    94
    95    /// Initializes the cache.
    96    #[inline]
    97    fn initialize(&self, value: usize) -> usize {
    98        debug_assert_eq!((value & !Cache::MASK), 0);
    99        self.0.store(value | Cache::INITIALIZED_BIT, Ordering::Relaxed);
    100        value
    101    }
    102}
    103
    104cfg_select! {
    105    feature = "std_detect_env_override" => {
    106        #[inline]
    107        fn disable_features(disable: &[u8], value: &mut Initializer) {
    108            if let Ok(disable) = core::str::from_utf8(disable) {
    109                for v in disable.split(" ") {
    110                    let _ = super::Feature::from_str(v).map(|v| value.unset(v as u32));
    111                }
    112            }
    113        }
    114
    115        #[inline]
    116        fn initialize(mut value: Initializer) -> Initializer {
    117            use core::ffi::CStr;
    118            const RUST_STD_DETECT_UNSTABLE: &CStr = c"RUST_STD_DETECT_UNSTABLE";
    119            cfg_select! {
    120                windows => {
    121                    use alloc::vec;
    122                    #[link(name = "kernel32")]
    123                    unsafe extern "system" {
    124                        fn GetEnvironmentVariableA(name: *const u8, buffer: *mut u8, size: u32) -> u32;
    125                    }
    126                    let len = unsafe { GetEnvironmentVariableA(RUST_STD_DETECT_UNSTABLE.as_ptr().cast::<u8>(), core::ptr::null_mut(), 0) };
    127                    if len > 0 {
    128                        // +1 to include the null terminator.
    129                        let mut env = vec![0; len as usize + 1];
    130                        let len = unsafe { GetEnvironmentVariableA(RUST_STD_DETECT_UNSTABLE.as_ptr().cast::<u8>(), env.as_mut_ptr(), len + 1) };
    131                        if len > 0 {
    132                            disable_features(&env[..len as usize], &mut value);
    133                        }
    134                    }
    135                }
    136                _ => {
    137                    let env = unsafe {
    138                        libc::getenv(RUST_STD_DETECT_UNSTABLE.as_ptr())
    139                    };
    140                    if !env.is_null() {
    141                        let len = unsafe { libc::strlen(env) };
    142                        let env = unsafe { core::slice::from_raw_parts(env as *const u8, len) };
    143                        disable_features(env, &mut value);
    144                    }
    145                }
    146            }
    147            do_initialize(value);
    148            value
    149        }
    150    }
    151    _ => {
    152        #[inline]
    153        fn initialize(value: Initializer) -> Initializer {
    154            do_initialize(value);
    155            value
    156        }
    157    }
    158}
    159
    160#[inline]
    161fn do_initialize(value: Initializer) {
    162    CACHE[0].initialize((value.0) as usize & Cache::MASK);
    163    CACHE[1].initialize((value.0 >> Cache::CAPACITY) as usize & Cache::MASK);
    164    CACHE[2].initialize((value.0 >> (2 * Cache::CAPACITY)) as usize & Cache::MASK);
    165}
    166
    167// We only have to detect features once, and it's fairly costly, so hint to LLVM
    168// that it should assume that cache hits are more common than misses (which is
    169// the point of caching). It's possibly unfortunate that this function needs to
    170// reach across modules like this to call `os::detect_features`, but it produces
    171// the best code out of several attempted variants.
    172//
    173// The `Initializer` that the cache was initialized with is returned, so that
    174// the caller can call `test()` on it without having to load the value from the
    175// cache again.
    176#[cold]
    177fn detect_and_initialize() -> Initializer {
    178    initialize(super::os::detect_features())
    179}
    180
    181/// Tests the `bit` of the storage. If the storage has not been initialized,
    182/// initializes it with the result of `os::detect_features()`.
    183///
    184/// On its first invocation, it detects the CPU features and caches them in the
    185/// `CACHE` global variable as an `AtomicU64`.
    186///
    187/// It uses the `Feature` variant to index into this variable as a bitset. If
    188/// the bit is set, the feature is enabled, and otherwise it is disabled.
    189///
    190/// If the feature `std_detect_env_override` is enabled looks for the env
    191/// variable `RUST_STD_DETECT_UNSTABLE` and uses its content to disable
    192/// Features that would had been otherwise detected.
    193#[inline]
    194pub(crate) fn test(bit: u32) -> bool {
    195    let (relative_bit, idx) = if bit < Cache::CAPACITY {
    196        (bit, 0)
    197    } else if bit < 2 * Cache::CAPACITY {
    198        (bit - Cache::CAPACITY, 1)
    199    } else {
    200        (bit - 2 * Cache::CAPACITY, 2)
    201    };
    202    CACHE[idx].test(relative_bit).unwrap_or_else(|| detect_and_initialize().test(bit))
    203}