simd.rs - source
Description: Source of the Rust file `library/core/src/intrinsics/simd.rs`.

#
core/intrinsics/
simd.rs

    1//! SIMD compiler intrinsics.
    2//!
    3//! In this module, a "vector" is any `repr(simd)` type.
    4
    5use crate::marker::ConstParamTy;
    6
    7/// Inserts an element into a vector, returning the updated vector.
    8///
    9/// `T` must be a vector with element type `U`, and `idx` must be `const`.
    10///
    11/// # Safety
    12///
    13/// `idx` must be in-bounds of the vector.
    14#[rustc_intrinsic]
    15#[rustc_nounwind]
    16pub const unsafe fn simd_insert<T, U>(x: T, idx: u32, val: U) -> T;
    17
    18/// Extracts an element from a vector.
    19///
    20/// `T` must be a vector with element type `U`, and `idx` must be `const`.
    21///
    22/// # Safety
    23///
    24/// `idx` must be const and in-bounds of the vector.
    25#[rustc_intrinsic]
    26#[rustc_nounwind]
    27pub const unsafe fn simd_extract<T, U>(x: T, idx: u32) -> U;
    28
    29/// Inserts an element into a vector, returning the updated vector.
    30///
    31/// `T` must be a vector with element type `U`.
    32///
    33/// If the index is `const`, [`simd_insert`] may emit better assembly.
    34///
    35/// # Safety
    36///
    37/// `idx` must be in-bounds of the vector.
    38#[rustc_nounwind]
    39#[rustc_intrinsic]
    40pub unsafe fn simd_insert_dyn<T, U>(mut x: T, idx: u32, val: U) -> T {
    41    // SAFETY: `idx` must be in-bounds
    42    unsafe { (&raw mut x).cast::<U>().add(idx as usize).write(val) }
    43    x
    44}
    45
    46/// Extracts an element from a vector.
    47///
    48/// `T` must be a vector with element type `U`.
    49///
    50/// If the index is `const`, [`simd_extract`] may emit better assembly.
    51///
    52/// # Safety
    53///
    54/// `idx` must be in-bounds of the vector.
    55#[rustc_nounwind]
    56#[rustc_intrinsic]
    57pub unsafe fn simd_extract_dyn<T, U>(x: T, idx: u32) -> U {
    58    // SAFETY: `idx` must be in-bounds
    59    unsafe { (&raw const x).cast::<U>().add(idx as usize).read() }
    60}
    61
    62/// Adds two simd vectors elementwise.
    63///
    64/// `T` must be a vector of integers or floats.
    65#[rustc_intrinsic]
    66#[rustc_nounwind]
    67pub const unsafe fn simd_add<T>(x: T, y: T) -> T;
    68
    69/// Subtracts `rhs` from `lhs` elementwise.
    70///
    71/// `T` must be a vector of integers or floats.
    72#[rustc_intrinsic]
    73#[rustc_nounwind]
    74pub const unsafe fn simd_sub<T>(lhs: T, rhs: T) -> T;
    75
    76/// Multiplies two simd vectors elementwise.
    77///
    78/// `T` must be a vector of integers or floats.
    79#[rustc_intrinsic]
    80#[rustc_nounwind]
    81pub const unsafe fn simd_mul<T>(x: T, y: T) -> T;
    82
    83/// Divides `lhs` by `rhs` elementwise.
    84///
    85/// `T` must be a vector of integers or floats.
    86///
    87/// # Safety
    88/// For integers, `rhs` must not contain any zero elements.
    89/// Additionally for signed integers, `<int>::MIN / -1` is undefined behavior.
    90#[rustc_intrinsic]
    91#[rustc_nounwind]
    92pub const unsafe fn simd_div<T>(lhs: T, rhs: T) -> T;
    93
    94/// Returns remainder of two vectors elementwise.
    95///
    96/// `T` must be a vector of integers or floats.
    97///
    98/// # Safety
    99/// For integers, `rhs` must not contain any zero elements.
    100/// Additionally for signed integers, `<int>::MIN / -1` is undefined behavior.
    101#[rustc_intrinsic]
    102#[rustc_nounwind]
    103pub const unsafe fn simd_rem<T>(lhs: T, rhs: T) -> T;
    104
    105/// Shifts vector left elementwise, with UB on overflow.
    106///
    107/// Shifts `lhs` left by `rhs`, shifting in sign bits for signed types.
    108///
    109/// `T` must be a vector of integers.
    110///
    111/// # Safety
    112///
    113/// Each element of `rhs` must be less than `<int>::BITS`.
    114#[rustc_intrinsic]
    115#[rustc_nounwind]
    116pub const unsafe fn simd_shl<T>(lhs: T, rhs: T) -> T;
    117
    118/// Shifts vector right elementwise, with UB on overflow.
    119///
    120/// `T` must be a vector of integers.
    121///
    122/// Shifts `lhs` right by `rhs`, shifting in sign bits for signed types.
    123///
    124/// # Safety
    125///
    126/// Each element of `rhs` must be less than `<int>::BITS`.
    127#[rustc_intrinsic]
    128#[rustc_nounwind]
    129pub const unsafe fn simd_shr<T>(lhs: T, rhs: T) -> T;
    130
    131/// Funnel Shifts vector left elementwise, with UB on overflow.
    132///
    133/// Concatenates `a` and `b` elementwise (with `a` in the most significant half),
    134/// creating a vector of the same length, but with each element being twice as
    135/// wide. Then shift this vector left elementwise by `shift`, shifting in zeros,
    136/// and extract the most significant half of each of the elements. If `a` and `b`
    137/// are the same, this is equivalent to an elementwise rotate left operation.
    138///
    139/// `T` must be a vector of integers.
    140///
    141/// # Safety
    142///
    143/// Each element of `shift` must be less than `<int>::BITS`.
    144#[rustc_intrinsic]
    145#[rustc_nounwind]
    146pub const unsafe fn simd_funnel_shl<T>(a: T, b: T, shift: T) -> T;
    147
    148/// Funnel Shifts vector right elementwise, with UB on overflow.
    149///
    150/// Concatenates `a` and `b` elementwise (with `a` in the most significant half),
    151/// creating a vector of the same length, but with each element being twice as
    152/// wide. Then shift this vector right elementwise by `shift`, shifting in zeros,
    153/// and extract the least significant half of each of the elements. If `a` and `b`
    154/// are the same, this is equivalent to an elementwise rotate right operation.
    155///
    156/// `T` must be a vector of integers.
    157///
    158/// # Safety
    159///
    160/// Each element of `shift` must be less than `<int>::BITS`.
    161#[rustc_intrinsic]
    162#[rustc_nounwind]
    163pub const unsafe fn simd_funnel_shr<T>(a: T, b: T, shift: T) -> T;
    164
    165/// "And"s vectors elementwise.
    166///
    167/// `T` must be a vector of integers.
    168#[rustc_intrinsic]
    169#[rustc_nounwind]
    170pub const unsafe fn simd_and<T>(x: T, y: T) -> T;
    171
    172/// "Ors" vectors elementwise.
    173///
    174/// `T` must be a vector of integers.
    175#[rustc_intrinsic]
    176#[rustc_nounwind]
    177pub const unsafe fn simd_or<T>(x: T, y: T) -> T;
    178
    179/// "Exclusive ors" vectors elementwise.
    180///
    181/// `T` must be a vector of integers.
    182#[rustc_intrinsic]
    183#[rustc_nounwind]
    184pub const unsafe fn simd_xor<T>(x: T, y: T) -> T;
    185
    186/// Numerically casts a vector, elementwise.
    187///
    188/// `T` and `U` must be vectors of integers or floats, and must have the same length.
    189///
    190/// When casting floats to integers, the result is truncated. Out-of-bounds result lead to UB.
    191/// When casting integers to floats, the result is rounded.
    192/// Otherwise, truncates or extends the value, maintaining the sign for signed integers.
    193///
    194/// # Safety
    195/// Casting from integer types is always safe.
    196/// Casting between two float types is also always safe.
    197///
    198/// Casting floats to integers truncates, following the same rules as `to_int_unchecked`.
    199/// Specifically, each element must:
    200/// * Not be `NaN`
    201/// * Not be infinite
    202/// * Be representable in the return type, after truncating off its fractional part
    203#[rustc_intrinsic]
    204#[rustc_nounwind]
    205pub const unsafe fn simd_cast<T, U>(x: T) -> U;
    206
    207/// Numerically casts a vector, elementwise.
    208///
    209/// `T` and `U` be a vectors of integers or floats, and must have the same length.
    210///
    211/// Like `simd_cast`, but saturates float-to-integer conversions (NaN becomes 0).
    212/// This matches regular `as` and is always safe.
    213///
    214/// When casting floats to integers, the result is truncated.
    215/// When casting integers to floats, the result is rounded.
    216/// Otherwise, truncates or extends the value, maintaining the sign for signed integers.
    217#[rustc_intrinsic]
    218#[rustc_nounwind]
    219pub const unsafe fn simd_as<T, U>(x: T) -> U;
    220
    221/// Negates a vector elementwise.
    222///
    223/// `T` must be a vector of integers or floats.
    224///
    225/// Rust panics for `-<int>::Min` due to overflow, but it is not UB with this intrinsic.
    226#[rustc_intrinsic]
    227#[rustc_nounwind]
    228pub const unsafe fn simd_neg<T>(x: T) -> T;
    229
    230/// Returns absolute value of a vector, elementwise.
    231///
    232/// `T` must be a vector of floating-point primitive types.
    233#[rustc_intrinsic]
    234#[rustc_nounwind]
    235pub const unsafe fn simd_fabs<T>(x: T) -> T;
    236
    237/// Returns the minimum of two vectors, elementwise.
    238///
    239/// `T` must be a vector of floating-point primitive types.
    240///
    241/// Follows IEEE-754 `minNum` semantics.
    242#[rustc_intrinsic]
    243#[rustc_nounwind]
    244pub const unsafe fn simd_fmin<T>(x: T, y: T) -> T;
    245
    246/// Returns the maximum of two vectors, elementwise.
    247///
    248/// `T` must be a vector of floating-point primitive types.
    249///
    250/// Follows IEEE-754 `maxNum` semantics.
    251#[rustc_intrinsic]
    252#[rustc_nounwind]
    253pub const unsafe fn simd_fmax<T>(x: T, y: T) -> T;
    254
    255/// Tests elementwise equality of two vectors.
    256///
    257/// `T` must be a vector of integers or floats.
    258///
    259/// `U` must be a vector of integers with the same number of elements and element size as `T`.
    260///
    261/// Returns `0` for false and `!0` for true.
    262#[rustc_intrinsic]
    263#[rustc_nounwind]
    264pub const unsafe fn simd_eq<T, U>(x: T, y: T) -> U;
    265
    266/// Tests elementwise inequality equality of two vectors.
    267///
    268/// `T` must be a vector of integers or floats.
    269///
    270/// `U` must be a vector of integers with the same number of elements and element size as `T`.
    271///
    272/// Returns `0` for false and `!0` for true.
    273#[rustc_intrinsic]
    274#[rustc_nounwind]
    275pub const unsafe fn simd_ne<T, U>(x: T, y: T) -> U;
    276
    277/// Tests if `x` is less than `y`, elementwise.
    278///
    279/// `T` must be a vector of integers or floats.
    280///
    281/// `U` must be a vector of integers with the same number of elements and element size as `T`.
    282///
    283/// Returns `0` for false and `!0` for true.
    284#[rustc_intrinsic]
    285#[rustc_nounwind]
    286pub const unsafe fn simd_lt<T, U>(x: T, y: T) -> U;
    287
    288/// Tests if `x` is less than or equal to `y`, elementwise.
    289///
    290/// `T` must be a vector of integers or floats.
    291///
    292/// `U` must be a vector of integers with the same number of elements and element size as `T`.
    293///
    294/// Returns `0` for false and `!0` for true.
    295#[rustc_intrinsic]
    296#[rustc_nounwind]
    297pub const unsafe fn simd_le<T, U>(x: T, y: T) -> U;
    298
    299/// Tests if `x` is greater than `y`, elementwise.
    300///
    301/// `T` must be a vector of integers or floats.
    302///
    303/// `U` must be a vector of integers with the same number of elements and element size as `T`.
    304///
    305/// Returns `0` for false and `!0` for true.
    306#[rustc_intrinsic]
    307#[rustc_nounwind]
    308pub const unsafe fn simd_gt<T, U>(x: T, y: T) -> U;
    309
    310/// Tests if `x` is greater than or equal to `y`, elementwise.
    311///
    312/// `T` must be a vector of integers or floats.
    313///
    314/// `U` must be a vector of integers with the same number of elements and element size as `T`.
    315///
    316/// Returns `0` for false and `!0` for true.
    317#[rustc_intrinsic]
    318#[rustc_nounwind]
    319pub const unsafe fn simd_ge<T, U>(x: T, y: T) -> U;
    320
    321/// Shuffles two vectors by const indices.
    322///
    323/// `T` must be a vector.
    324///
    325/// `U` must be a **const** vector of `u32`s. This means it must either refer to a named
    326/// const or be given as an inline const expression (`const { ... }`).
    327///
    328/// `V` must be a vector with the same element type as `T` and the same length as `U`.
    329///
    330/// Returns a new vector such that element `i` is selected from `xy[idx[i]]`, where `xy`
    331/// is the concatenation of `x` and `y`. It is a compile-time error if `idx[i]` is out-of-bounds
    332/// of `xy`.
    333#[rustc_intrinsic]
    334#[rustc_nounwind]
    335pub const unsafe fn simd_shuffle<T, U, V>(x: T, y: T, idx: U) -> V;
    336
    337/// Reads a vector of pointers.
    338///
    339/// `T` must be a vector.
    340///
    341/// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`.
    342///
    343/// `V` must be a vector of integers with the same length as `T` (but any element size).
    344///
    345/// For each pointer in `ptr`, if the corresponding value in `mask` is `!0`, read the pointer.
    346/// Otherwise if the corresponding value in `mask` is `0`, return the corresponding value from
    347/// `val`.
    348///
    349/// # Safety
    350/// Unmasked values in `T` must be readable as if by `<ptr>::read` (e.g. aligned to the element
    351/// type).
    352///
    353/// `mask` must only contain `0` or `!0` values.
    354#[rustc_intrinsic]
    355#[rustc_nounwind]
    356pub const unsafe fn simd_gather<T, U, V>(val: T, ptr: U, mask: V) -> T;
    357
    358/// Writes to a vector of pointers.
    359///
    360/// `T` must be a vector.
    361///
    362/// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`.
    363///
    364/// `V` must be a vector of integers with the same length as `T` (but any element size).
    365///
    366/// For each pointer in `ptr`, if the corresponding value in `mask` is `!0`, write the
    367/// corresponding value in `val` to the pointer.
    368/// Otherwise if the corresponding value in `mask` is `0`, do nothing.
    369///
    370/// The stores happen in left-to-right order.
    371/// (This is relevant in case two of the stores overlap.)
    372///
    373/// # Safety
    374/// Unmasked values in `T` must be writeable as if by `<ptr>::write` (e.g. aligned to the element
    375/// type).
    376///
    377/// `mask` must only contain `0` or `!0` values.
    378#[rustc_intrinsic]
    379#[rustc_nounwind]
    380pub const unsafe fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V);
    381
    382/// A type for alignment options for SIMD masked load/store intrinsics.
    383#[derive(Debug, ConstParamTy, PartialEq, Eq)]
    384pub enum SimdAlign {
    385    // These values must match the compiler's `SimdAlign` defined in
    386    // `rustc_middle/src/ty/consts/int.rs`!
    387    /// No alignment requirements on the pointer
    388    Unaligned = 0,
    389    /// The pointer must be aligned to the element type of the SIMD vector
    390    Element = 1,
    391    /// The pointer must be aligned to the SIMD vector type
    392    Vector = 2,
    393}
    394
    395/// Reads a vector of pointers.
    396///
    397/// `T` must be a vector.
    398///
    399/// `U` must be a pointer to the element type of `T`
    400///
    401/// `V` must be a vector of integers with the same length as `T` (but any element size).
    402///
    403/// For each element, if the corresponding value in `mask` is `!0`, read the corresponding
    404/// pointer offset from `ptr`.
    405/// The first element is loaded from `ptr`, the second from `ptr.wrapping_offset(1)` and so on.
    406/// Otherwise if the corresponding value in `mask` is `0`, return the corresponding value from
    407/// `val`.
    408///
    409/// # Safety
    410/// `ptr` must be aligned according to the `ALIGN` parameter, see [`SimdAlign`] for details.
    411///
    412/// `mask` must only contain `0` or `!0` values.
    413#[rustc_intrinsic]
    414#[rustc_nounwind]
    415pub const unsafe fn simd_masked_load<V, U, T, const ALIGN: SimdAlign>(mask: V, ptr: U, val: T)
    416-> T;
    417
    418/// Writes to a vector of pointers.
    419///
    420/// `T` must be a vector.
    421///
    422/// `U` must be a pointer to the element type of `T`
    423///
    424/// `V` must be a vector of integers with the same length as `T` (but any element size).
    425///
    426/// For each element, if the corresponding value in `mask` is `!0`, write the corresponding
    427/// value in `val` to the pointer offset from `ptr`.
    428/// The first element is written to `ptr`, the second to `ptr.wrapping_offset(1)` and so on.
    429/// Otherwise if the corresponding value in `mask` is `0`, do nothing.
    430///
    431/// # Safety
    432/// `ptr` must be aligned according to the `ALIGN` parameter, see [`SimdAlign`] for details.
    433///
    434/// `mask` must only contain `0` or `!0` values.
    435#[rustc_intrinsic]
    436#[rustc_nounwind]
    437pub const unsafe fn simd_masked_store<V, U, T, const ALIGN: SimdAlign>(mask: V, ptr: U, val: T);
    438
    439/// Adds two simd vectors elementwise, with saturation.
    440///
    441/// `T` must be a vector of integer primitive types.
    442#[rustc_intrinsic]
    443#[rustc_nounwind]
    444pub const unsafe fn simd_saturating_add<T>(x: T, y: T) -> T;
    445
    446/// Subtracts two simd vectors elementwise, with saturation.
    447///
    448/// `T` must be a vector of integer primitive types.
    449///
    450/// Subtract `rhs` from `lhs`.
    451#[rustc_intrinsic]
    452#[rustc_nounwind]
    453pub const unsafe fn simd_saturating_sub<T>(lhs: T, rhs: T) -> T;
    454
    455/// Adds elements within a vector from left to right.
    456///
    457/// `T` must be a vector of integers or floats.
    458///
    459/// `U` must be the element type of `T`.
    460///
    461/// Starting with the value `y`, add the elements of `x` and accumulate.
    462#[rustc_intrinsic]
    463#[rustc_nounwind]
    464pub const unsafe fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U;
    465
    466/// Adds elements within a vector in arbitrary order. May also be re-associated with
    467/// unordered additions on the inputs/outputs.
    468///
    469/// `T` must be a vector of integers or floats.
    470///
    471/// `U` must be the element type of `T`.
    472#[rustc_intrinsic]
    473#[rustc_nounwind]
    474pub unsafe fn simd_reduce_add_unordered<T, U>(x: T) -> U;
    475
    476/// Multiplies elements within a vector from left to right.
    477///
    478/// `T` must be a vector of integers or floats.
    479///
    480/// `U` must be the element type of `T`.
    481///
    482/// Starting with the value `y`, multiply the elements of `x` and accumulate.
    483#[rustc_intrinsic]
    484#[rustc_nounwind]
    485pub const unsafe fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U;
    486
    487/// Multiplies elements within a vector in arbitrary order. May also be re-associated with
    488/// unordered additions on the inputs/outputs.
    489///
    490/// `T` must be a vector of integers or floats.
    491///
    492/// `U` must be the element type of `T`.
    493#[rustc_intrinsic]
    494#[rustc_nounwind]
    495pub unsafe fn simd_reduce_mul_unordered<T, U>(x: T) -> U;
    496
    497/// Checks if all mask values are true.
    498///
    499/// `T` must be a vector of integer primitive types.
    500///
    501/// # Safety
    502/// `x` must contain only `0` or `!0`.
    503#[rustc_intrinsic]
    504#[rustc_nounwind]
    505pub const unsafe fn simd_reduce_all<T>(x: T) -> bool;
    506
    507/// Checks if any mask value is true.
    508///
    509/// `T` must be a vector of integer primitive types.
    510///
    511/// # Safety
    512/// `x` must contain only `0` or `!0`.
    513#[rustc_intrinsic]
    514#[rustc_nounwind]
    515pub const unsafe fn simd_reduce_any<T>(x: T) -> bool;
    516
    517/// Returns the maximum element of a vector.
    518///
    519/// `T` must be a vector of integers or floats.
    520///
    521/// `U` must be the element type of `T`.
    522///
    523/// For floating-point values, uses IEEE-754 `maxNum`.
    524#[rustc_intrinsic]
    525#[rustc_nounwind]
    526pub const unsafe fn simd_reduce_max<T, U>(x: T) -> U;
    527
    528/// Returns the minimum element of a vector.
    529///
    530/// `T` must be a vector of integers or floats.
    531///
    532/// `U` must be the element type of `T`.
    533///
    534/// For floating-point values, uses IEEE-754 `minNum`.
    535#[rustc_intrinsic]
    536#[rustc_nounwind]
    537pub const unsafe fn simd_reduce_min<T, U>(x: T) -> U;
    538
    539/// Logical "and"s all elements together.
    540///
    541/// `T` must be a vector of integers or floats.
    542///
    543/// `U` must be the element type of `T`.
    544#[rustc_intrinsic]
    545#[rustc_nounwind]
    546pub const unsafe fn simd_reduce_and<T, U>(x: T) -> U;
    547
    548/// Logical "ors" all elements together.
    549///
    550/// `T` must be a vector of integers or floats.
    551///
    552/// `U` must be the element type of `T`.
    553#[rustc_intrinsic]
    554#[rustc_nounwind]
    555pub const unsafe fn simd_reduce_or<T, U>(x: T) -> U;
    556
    557/// Logical "exclusive ors" all elements together.
    558///
    559/// `T` must be a vector of integers or floats.
    560///
    561/// `U` must be the element type of `T`.
    562#[rustc_intrinsic]
    563#[rustc_nounwind]
    564pub const unsafe fn simd_reduce_xor<T, U>(x: T) -> U;
    565
    566/// Truncates an integer vector to a bitmask.
    567///
    568/// `T` must be an integer vector.
    569///
    570/// `U` must be either the smallest unsigned integer with at least as many bits as the length
    571/// of `T`, or the smallest array of `u8` with at least as many bits as the length of `T`.
    572///
    573/// Each element is truncated to a single bit and packed into the result.
    574///
    575/// No matter whether the output is an array or an unsigned integer, it is treated as a single
    576/// contiguous list of bits. The bitmask is always packed on the least-significant side of the
    577/// output, and padded with 0s in the most-significant bits. The order of the bits depends on
    578/// endianness:
    579///
    580/// * On little endian, the least significant bit corresponds to the first vector element.
    581/// * On big endian, the least significant bit corresponds to the last vector element.
    582///
    583/// For example, `[!0, 0, !0, !0]` packs to
    584/// - `0b1101u8` or `[0b1101]` on little endian, and
    585/// - `0b1011u8` or `[0b1011]` on big endian.
    586///
    587/// To consider a larger example,
    588/// `[!0, 0, 0, 0, 0, 0, 0, 0, !0, !0, 0, 0, 0, 0, !0, 0]` packs to
    589/// - `0b0100001100000001u16` or `[0b00000001, 0b01000011]` on little endian, and
    590/// - `0b1000000011000010u16` or `[0b10000000, 0b11000010]` on big endian.
    591///
    592/// And finally, a non-power-of-2 example with multiple bytes:
    593/// `[!0, !0, 0, !0, 0, 0, !0, 0, !0, 0]` packs to
    594/// - `0b0101001011u16` or `[0b01001011, 0b01]` on little endian, and
    595/// - `0b1101001010u16` or `[0b11, 0b01001010]` on big endian.
    596///
    597/// # Safety
    598/// `x` must contain only `0` and `!0`.
    599#[rustc_intrinsic]
    600#[rustc_nounwind]
    601pub const unsafe fn simd_bitmask<T, U>(x: T) -> U;
    602
    603/// Selects elements from a mask.
    604///
    605/// `T` must be a vector.
    606///
    607/// `M` must be an integer vector with the same length as `T` (but any element size).
    608///
    609/// For each element, if the corresponding value in `mask` is `!0`, select the element from
    610/// `if_true`.  If the corresponding value in `mask` is `0`, select the element from
    611/// `if_false`.
    612///
    613/// # Safety
    614/// `mask` must only contain `0` and `!0`.
    615#[rustc_intrinsic]
    616#[rustc_nounwind]
    617pub const unsafe fn simd_select<M, T>(mask: M, if_true: T, if_false: T) -> T;
    618
    619/// Selects elements from a bitmask.
    620///
    621/// `M` must be an unsigned integer or array of `u8`, matching `simd_bitmask`.
    622///
    623/// `T` must be a vector.
    624///
    625/// For each element, if the bit in `mask` is `1`, select the element from
    626/// `if_true`.  If the corresponding bit in `mask` is `0`, select the element from
    627/// `if_false`.
    628/// The remaining bits of the mask are ignored.
    629///
    630/// The bitmask bit order matches `simd_bitmask`.
    631#[rustc_intrinsic]
    632#[rustc_nounwind]
    633pub const unsafe fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T;
    634
    635/// Calculates the offset from a pointer vector elementwise, potentially
    636/// wrapping.
    637///
    638/// `T` must be a vector of pointers.
    639///
    640/// `U` must be a vector of `isize` or `usize` with the same number of elements as `T`.
    641///
    642/// Operates as if by `<ptr>::wrapping_offset`.
    643#[rustc_intrinsic]
    644#[rustc_nounwind]
    645pub const unsafe fn simd_arith_offset<T, U>(ptr: T, offset: U) -> T;
    646
    647/// Casts a vector of pointers.
    648///
    649/// `T` and `U` must be vectors of pointers with the same number of elements.
    650#[rustc_intrinsic]
    651#[rustc_nounwind]
    652pub const unsafe fn simd_cast_ptr<T, U>(ptr: T) -> U;
    653
    654/// Exposes a vector of pointers as a vector of addresses.
    655///
    656/// `T` must be a vector of pointers.
    657///
    658/// `U` must be a vector of `usize` with the same length as `T`.
    659#[rustc_intrinsic]
    660#[rustc_nounwind]
    661pub unsafe fn simd_expose_provenance<T, U>(ptr: T) -> U;
    662
    663/// Creates a vector of pointers from a vector of addresses.
    664///
    665/// `T` must be a vector of `usize`.
    666///
    667/// `U` must be a vector of pointers, with the same length as `T`.
    668#[rustc_intrinsic]
    669#[rustc_nounwind]
    670pub const unsafe fn simd_with_exposed_provenance<T, U>(addr: T) -> U;
    671
    672/// Swaps bytes of each element.
    673///
    674/// `T` must be a vector of integers.
    675#[rustc_intrinsic]
    676#[rustc_nounwind]
    677pub const unsafe fn simd_bswap<T>(x: T) -> T;
    678
    679/// Reverses bits of each element.
    680///
    681/// `T` must be a vector of integers.
    682#[rustc_intrinsic]
    683#[rustc_nounwind]
    684pub const unsafe fn simd_bitreverse<T>(x: T) -> T;
    685
    686/// Counts the leading zeros of each element.
    687///
    688/// `T` must be a vector of integers.
    689#[rustc_intrinsic]
    690#[rustc_nounwind]
    691pub const unsafe fn simd_ctlz<T>(x: T) -> T;
    692
    693/// Counts the number of ones in each element.
    694///
    695/// `T` must be a vector of integers.
    696#[rustc_intrinsic]
    697#[rustc_nounwind]
    698pub const unsafe fn simd_ctpop<T>(x: T) -> T;
    699
    700/// Counts the trailing zeros of each element.
    701///
    702/// `T` must be a vector of integers.
    703#[rustc_intrinsic]
    704#[rustc_nounwind]
    705pub const unsafe fn simd_cttz<T>(x: T) -> T;
    706
    707/// Rounds up each element to the next highest integer-valued float.
    708///
    709/// `T` must be a vector of floats.
    710#[rustc_intrinsic]
    711#[rustc_nounwind]
    712pub const unsafe fn simd_ceil<T>(x: T) -> T;
    713
    714/// Rounds down each element to the next lowest integer-valued float.
    715///
    716/// `T` must be a vector of floats.
    717#[rustc_intrinsic]
    718#[rustc_nounwind]
    719pub const unsafe fn simd_floor<T>(x: T) -> T;
    720
    721/// Rounds each element to the closest integer-valued float.
    722/// Ties are resolved by rounding away from 0.
    723///
    724/// `T` must be a vector of floats.
    725#[rustc_intrinsic]
    726#[rustc_nounwind]
    727pub const unsafe fn simd_round<T>(x: T) -> T;
    728
    729/// Rounds each element to the closest integer-valued float.
    730/// Ties are resolved by rounding to the number with an even least significant digit
    731///
    732/// `T` must be a vector of floats.
    733#[rustc_intrinsic]
    734#[rustc_nounwind]
    735pub const unsafe fn simd_round_ties_even<T>(x: T) -> T;
    736
    737/// Returns the integer part of each element as an integer-valued float.
    738/// In other words, non-integer values are truncated towards zero.
    739///
    740/// `T` must be a vector of floats.
    741#[rustc_intrinsic]
    742#[rustc_nounwind]
    743pub const unsafe fn simd_trunc<T>(x: T) -> T;
    744
    745/// Takes the square root of each element.
    746///
    747/// `T` must be a vector of floats.
    748#[rustc_intrinsic]
    749#[rustc_nounwind]
    750pub unsafe fn simd_fsqrt<T>(x: T) -> T;
    751
    752/// Computes `(x*y) + z` for each element, but without any intermediate rounding.
    753///
    754/// `T` must be a vector of floats.
    755#[rustc_intrinsic]
    756#[rustc_nounwind]
    757pub const unsafe fn simd_fma<T>(x: T, y: T, z: T) -> T;
    758
    759/// Computes `(x*y) + z` for each element, non-deterministically executing either
    760/// a fused multiply-add or two operations with rounding of the intermediate result.
    761///
    762/// The operation is fused if the code generator determines that target instruction
    763/// set has support for a fused operation, and that the fused operation is more efficient
    764/// than the equivalent, separate pair of mul and add instructions. It is unspecified
    765/// whether or not a fused operation is selected, and that may depend on optimization
    766/// level and context, for example. It may even be the case that some SIMD lanes get fused
    767/// and others do not.
    768///
    769/// `T` must be a vector of floats.
    770#[rustc_intrinsic]
    771#[rustc_nounwind]
    772pub const unsafe fn simd_relaxed_fma<T>(x: T, y: T, z: T) -> T;
    773
    774// Computes the sine of each element.
    775///
    776/// `T` must be a vector of floats.
    777#[rustc_intrinsic]
    778#[rustc_nounwind]
    779pub unsafe fn simd_fsin<T>(a: T) -> T;
    780
    781// Computes the cosine of each element.
    782///
    783/// `T` must be a vector of floats.
    784#[rustc_intrinsic]
    785#[rustc_nounwind]
    786pub unsafe fn simd_fcos<T>(a: T) -> T;
    787
    788// Computes the exponential function of each element.
    789///
    790/// `T` must be a vector of floats.
    791#[rustc_intrinsic]
    792#[rustc_nounwind]
    793pub unsafe fn simd_fexp<T>(a: T) -> T;
    794
    795// Computes 2 raised to the power of each element.
    796///
    797/// `T` must be a vector of floats.
    798#[rustc_intrinsic]
    799#[rustc_nounwind]
    800pub unsafe fn simd_fexp2<T>(a: T) -> T;
    801
    802// Computes the base 10 logarithm of each element.
    803///
    804/// `T` must be a vector of floats.
    805#[rustc_intrinsic]
    806#[rustc_nounwind]
    807pub unsafe fn simd_flog10<T>(a: T) -> T;
    808
    809// Computes the base 2 logarithm of each element.
    810///
    811/// `T` must be a vector of floats.
    812#[rustc_intrinsic]
    813#[rustc_nounwind]
    814pub unsafe fn simd_flog2<T>(a: T) -> T;
    815
    816// Computes the natural logarithm of each element.
    817///
    818/// `T` must be a vector of floats.
    819#[rustc_intrinsic]
    820#[rustc_nounwind]
    821pub unsafe fn simd_flog<T>(a: T) -> T;