wtf8.rs - source
Description: Source of the Rust file `library/core/src/wtf8.rs`.

#
core/
wtf8.rs

    1//! Implementation of [the WTF-8 encoding](https://simonsapin.github.io/wtf-8/).
    2//!
    3//! This library uses Rust’s type system to maintain
    4//! [well-formedness](https://simonsapin.github.io/wtf-8/#well-formed),
    5//! like the `String` and `&str` types do for UTF-8.
    6//!
    7//! Since [WTF-8 must not be used
    8//! for interchange](https://simonsapin.github.io/wtf-8/#intended-audience),
    9//! this library deliberately does not provide access to the underlying bytes
    10//! of WTF-8 strings,
    11//! nor can it decode WTF-8 from arbitrary bytes.
    12//! WTF-8 strings can be obtained from UTF-8, UTF-16, or code points.
    13#![unstable(
    14    feature = "wtf8_internals",
    15    issue = "none",
    16    reason = "this is internal code for representing OsStr on some platforms and not a public API"
    17)]
    18// rustdoc bug: doc(hidden) on the module won't stop types in the module from showing up in trait
    19// implementations, so, we'll have to add more doc(hidden)s anyway
    20#![doc(hidden)]
    21
    22use crate::char::{EscapeDebugExtArgs, encode_utf16_raw};
    23use crate::clone::CloneToUninit;
    24use crate::fmt::{self, Write};
    25use crate::hash::{Hash, Hasher};
    26use crate::iter::FusedIterator;
    27use crate::num::niche_types::CodePointInner;
    28use crate::str::next_code_point;
    29use crate::{ops, slice, str};
    30
    31/// A Unicode code point: from U+0000 to U+10FFFF.
    32///
    33/// Compares with the `char` type,
    34/// which represents a Unicode scalar value:
    35/// a code point that is not a surrogate (U+D800 to U+DFFF).
    36#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Copy)]
    37#[doc(hidden)]
    38pub struct CodePoint(CodePointInner);
    39
    40/// Format the code point as `U+` followed by four to six hexadecimal digits.
    41/// Example: `U+1F4A9`
    42impl fmt::Debug for CodePoint {
    43    #[inline]
    44    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    45        write!(formatter, "U+{:04X}", self.0.as_inner())
    46    }
    47}
    48
    49impl CodePoint {
    50    /// Unsafely creates a new `CodePoint` without checking the value.
    51    ///
    52    /// Only use when `value` is known to be less than or equal to 0x10FFFF.
    53    #[inline]
    54    pub unsafe fn from_u32_unchecked(value: u32) -> CodePoint {
    55        // SAFETY: Guaranteed by caller.
    56        CodePoint(unsafe { CodePointInner::new_unchecked(value) })
    57    }
    58
    59    /// Creates a new `CodePoint` if the value is a valid code point.
    60    ///
    61    /// Returns `None` if `value` is above 0x10FFFF.
    62    #[inline]
    63    pub fn from_u32(value: u32) -> Option<CodePoint> {
    64        Some(CodePoint(CodePointInner::new(value)?))
    65    }
    66
    67    /// Creates a new `CodePoint` from a `char`.
    68    ///
    69    /// Since all Unicode scalar values are code points, this always succeeds.
    70    #[inline]
    71    pub fn from_char(value: char) -> CodePoint {
    72        // SAFETY: All char are valid for this type.
    73        unsafe { CodePoint::from_u32_unchecked(value as u32) }
    74    }
    75
    76    /// Returns the numeric value of the code point.
    77    #[inline]
    78    pub fn to_u32(&self) -> u32 {
    79        self.0.as_inner()
    80    }
    81
    82    /// Returns the numeric value of the code point if it is a leading surrogate.
    83    #[inline]
    84    pub fn to_lead_surrogate(&self) -> Option<u16> {
    85        match self.to_u32() {
    86            lead @ 0xD800..=0xDBFF => Some(lead as u16),
    87            _ => None,
    88        }
    89    }
    90
    91    /// Returns the numeric value of the code point if it is a trailing surrogate.
    92    #[inline]
    93    pub fn to_trail_surrogate(&self) -> Option<u16> {
    94        match self.to_u32() {
    95            trail @ 0xDC00..=0xDFFF => Some(trail as u16),
    96            _ => None,
    97        }
    98    }
    99
    100    /// Optionally returns a Unicode scalar value for the code point.
    101    ///
    102    /// Returns `None` if the code point is a surrogate (from U+D800 to U+DFFF).
    103    #[inline]
    104    pub fn to_char(&self) -> Option<char> {
    105        match self.to_u32() {
    106            0xD800..=0xDFFF => None,
    107            // SAFETY: We explicitly check that the char is valid.
    108            valid => Some(unsafe { char::from_u32_unchecked(valid) }),
    109        }
    110    }
    111
    112    /// Returns a Unicode scalar value for the code point.
    113    ///
    114    /// Returns `'\u{FFFD}'` (the replacement character “�”)
    115    /// if the code point is a surrogate (from U+D800 to U+DFFF).
    116    #[inline]
    117    pub fn to_char_lossy(&self) -> char {
    118        self.to_char().unwrap_or(char::REPLACEMENT_CHARACTER)
    119    }
    120}
    121
    122/// A borrowed slice of well-formed WTF-8 data.
    123///
    124/// Similar to `&str`, but can additionally contain surrogate code points
    125/// if they’re not in a surrogate pair.
    126#[derive(Eq, Ord, PartialEq, PartialOrd)]
    127#[repr(transparent)]
    128#[rustc_has_incoherent_inherent_impls]
    129#[doc(hidden)]
    130pub struct Wtf8 {
    131    bytes: [u8],
    132}
    133
    134impl AsRef<[u8]> for Wtf8 {
    135    #[inline]
    136    fn as_ref(&self) -> &[u8] {
    137        &self.bytes
    138    }
    139}
    140
    141/// Formats the string in double quotes, with characters escaped according to
    142/// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`,
    143/// where each `x` is a hexadecimal digit.
    144impl fmt::Debug for Wtf8 {
    145    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    146        fn write_str_escaped(f: &mut fmt::Formatter<'_>, s: &str) -> fmt::Result {
    147            use crate::fmt::Write as _;
    148            for c in s.chars().flat_map(|c| {
    149                c.escape_debug_ext(EscapeDebugExtArgs {
    150                    escape_grapheme_extended: true,
    151                    escape_single_quote: false,
    152                    escape_double_quote: true,
    153                })
    154            }) {
    155                f.write_char(c)?
    156            }
    157            Ok(())
    158        }
    159
    160        formatter.write_char('"')?;
    161        let mut pos = 0;
    162        while let Some((surrogate_pos, surrogate)) = self.next_surrogate(pos) {
    163            // SAFETY: next_surrogate provides an index for a range of valid UTF-8 bytes.
    164            write_str_escaped(formatter, unsafe {
    165                str::from_utf8_unchecked(&self.bytes[pos..surrogate_pos])
    166            })?;
    167            write!(formatter, "\\u{{{:x}}}", surrogate)?;
    168            pos = surrogate_pos + 3;
    169        }
    170
    171        // SAFETY: after next_surrogate returns None, the remainder is valid UTF-8.
    172        write_str_escaped(formatter, unsafe { str::from_utf8_unchecked(&self.bytes[pos..]) })?;
    173        formatter.write_char('"')
    174    }
    175}
    176
    177/// Formats the string with unpaired surrogates substituted with the replacement
    178/// character, U+FFFD.
    179impl fmt::Display for Wtf8 {
    180    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    181        let wtf8_bytes = &self.bytes;
    182        let mut pos = 0;
    183        loop {
    184            match self.next_surrogate(pos) {
    185                Some((surrogate_pos, _)) => {
    186                    // SAFETY: next_surrogate provides an index for a range of valid UTF-8 bytes.
    187                    formatter.write_str(unsafe {
    188                        str::from_utf8_unchecked(&wtf8_bytes[pos..surrogate_pos])
    189                    })?;
    190                    formatter.write_char(char::REPLACEMENT_CHARACTER)?;
    191                    pos = surrogate_pos + 3;
    192                }
    193                None => {
    194                    // SAFETY: after next_surrogate returns None, the remainder is valid UTF-8.
    195                    let s = unsafe { str::from_utf8_unchecked(&wtf8_bytes[pos..]) };
    196                    if pos == 0 { return s.fmt(formatter) } else { return formatter.write_str(s) }
    197                }
    198            }
    199        }
    200    }
    201}
    202
    203impl Wtf8 {
    204    /// Creates a WTF-8 slice from a UTF-8 `&str` slice.
    205    #[inline]
    206    pub fn from_str(value: &str) -> &Wtf8 {
    207        // SAFETY: Since WTF-8 is a superset of UTF-8, this always is valid.
    208        unsafe { Wtf8::from_bytes_unchecked(value.as_bytes()) }
    209    }
    210
    211    /// Creates a WTF-8 slice from a WTF-8 byte slice.
    212    ///
    213    /// Since the byte slice is not checked for valid WTF-8, this functions is
    214    /// marked unsafe.
    215    #[inline]
    216    pub unsafe fn from_bytes_unchecked(value: &[u8]) -> &Wtf8 {
    217        // SAFETY: start with &[u8], end with fancy &[u8]
    218        unsafe { &*(value as *const [u8] as *const Wtf8) }
    219    }
    220
    221    /// Creates a mutable WTF-8 slice from a mutable WTF-8 byte slice.
    222    ///
    223    /// Since the byte slice is not checked for valid WTF-8, this functions is
    224    /// marked unsafe.
    225    #[inline]
    226    pub unsafe fn from_mut_bytes_unchecked(value: &mut [u8]) -> &mut Wtf8 {
    227        // SAFETY: start with &mut [u8], end with fancy &mut [u8]
    228        unsafe { &mut *(value as *mut [u8] as *mut Wtf8) }
    229    }
    230
    231    /// Returns the length, in WTF-8 bytes.
    232    #[inline]
    233    pub fn len(&self) -> usize {
    234        self.bytes.len()
    235    }
    236
    237    #[inline]
    238    pub fn is_empty(&self) -> bool {
    239        self.bytes.is_empty()
    240    }
    241
    242    /// Returns the code point at `position` if it is in the ASCII range,
    243    /// or `b'\xFF'` otherwise.
    244    ///
    245    /// # Panics
    246    ///
    247    /// Panics if `position` is beyond the end of the string.
    248    #[inline]
    249    pub fn ascii_byte_at(&self, position: usize) -> u8 {
    250        match self.bytes[position] {
    251            ascii_byte @ 0x00..=0x7F => ascii_byte,
    252            _ => 0xFF,
    253        }
    254    }
    255
    256    /// Returns an iterator for the string’s code points.
    257    #[inline]
    258    pub fn code_points(&self) -> Wtf8CodePoints<'_> {
    259        Wtf8CodePoints { bytes: self.bytes.iter() }
    260    }
    261
    262    /// Access raw bytes of WTF-8 data
    263    #[inline]
    264    pub fn as_bytes(&self) -> &[u8] {
    265        &self.bytes
    266    }
    267
    268    /// Tries to convert the string to UTF-8 and return a `&str` slice.
    269    ///
    270    /// Returns `None` if the string contains surrogates.
    271    ///
    272    /// This does not copy the data.
    273    #[inline]
    274    pub fn as_str(&self) -> Result<&str, str::Utf8Error> {
    275        str::from_utf8(&self.bytes)
    276    }
    277
    278    /// Converts the WTF-8 string to potentially ill-formed UTF-16
    279    /// and return an iterator of 16-bit code units.
    280    ///
    281    /// This is lossless:
    282    /// calling `Wtf8Buf::from_ill_formed_utf16` on the resulting code units
    283    /// would always return the original WTF-8 string.
    284    #[inline]
    285    pub fn encode_wide(&self) -> EncodeWide<'_> {
    286        EncodeWide { code_points: self.code_points(), extra: 0 }
    287    }
    288
    289    #[inline]
    290    pub fn next_surrogate(&self, mut pos: usize) -> Option<(usize, u16)> {
    291        let mut iter = self.bytes[pos..].iter();
    292        loop {
    293            let b = *iter.next()?;
    294            if b < 0x80 {
    295                pos += 1;
    296            } else if b < 0xE0 {
    297                iter.next();
    298                pos += 2;
    299            } else if b == 0xED {
    300                match (iter.next(), iter.next()) {
    301                    (Some(&b2), Some(&b3)) if b2 >= 0xA0 => {
    302                        return Some((pos, decode_surrogate(b2, b3)));
    303                    }
    304                    _ => pos += 3,
    305                }
    306            } else if b < 0xF0 {
    307                iter.next();
    308                iter.next();
    309                pos += 3;
    310            } else {
    311                iter.next();
    312                iter.next();
    313                iter.next();
    314                pos += 4;
    315            }
    316        }
    317    }
    318
    319    #[inline]
    320    pub fn final_lead_surrogate(&self) -> Option<u16> {
    321        match self.bytes {
    322            [.., 0xED, b2 @ 0xA0..=0xAF, b3] => Some(decode_surrogate(b2, b3)),
    323            _ => None,
    324        }
    325    }
    326
    327    #[inline]
    328    pub fn initial_trail_surrogate(&self) -> Option<u16> {
    329        match self.bytes {
    330            [0xED, b2 @ 0xB0..=0xBF, b3, ..] => Some(decode_surrogate(b2, b3)),
    331            _ => None,
    332        }
    333    }
    334
    335    #[inline]
    336    pub fn make_ascii_lowercase(&mut self) {
    337        self.bytes.make_ascii_lowercase()
    338    }
    339
    340    #[inline]
    341    pub fn make_ascii_uppercase(&mut self) {
    342        self.bytes.make_ascii_uppercase()
    343    }
    344
    345    #[inline]
    346    pub fn is_ascii(&self) -> bool {
    347        self.bytes.is_ascii()
    348    }
    349
    350    #[inline]
    351    pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool {
    352        self.bytes.eq_ignore_ascii_case(&other.bytes)
    353    }
    354}
    355
    356/// Returns a slice of the given string for the byte range \[`begin`..`end`).
    357///
    358/// # Panics
    359///
    360/// Panics when `begin` and `end` do not point to code point boundaries,
    361/// or point beyond the end of the string.
    362impl ops::Index<ops::Range<usize>> for Wtf8 {
    363    type Output = Wtf8;
    364
    365    #[inline]
    366    fn index(&self, range: ops::Range<usize>) -> &Wtf8 {
    367        if range.start <= range.end
    368            && self.is_code_point_boundary(range.start)
    369            && self.is_code_point_boundary(range.end)
    370        {
    371            // SAFETY: is_code_point_boundary checks that the index is valid
    372            unsafe { slice_unchecked(self, range.start, range.end) }
    373        } else {
    374            slice_error_fail(self, range.start, range.end)
    375        }
    376    }
    377}
    378
    379/// Returns a slice of the given string from byte `begin` to its end.
    380///
    381/// # Panics
    382///
    383/// Panics when `begin` is not at a code point boundary,
    384/// or is beyond the end of the string.
    385impl ops::Index<ops::RangeFrom<usize>> for Wtf8 {
    386    type Output = Wtf8;
    387
    388    #[inline]
    389    fn index(&self, range: ops::RangeFrom<usize>) -> &Wtf8 {
    390        if self.is_code_point_boundary(range.start) {
    391            // SAFETY: is_code_point_boundary checks that the index is valid
    392            unsafe { slice_unchecked(self, range.start, self.len()) }
    393        } else {
    394            slice_error_fail(self, range.start, self.len())
    395        }
    396    }
    397}
    398
    399/// Returns a slice of the given string from its beginning to byte `end`.
    400///
    401/// # Panics
    402///
    403/// Panics when `end` is not at a code point boundary,
    404/// or is beyond the end of the string.
    405impl ops::Index<ops::RangeTo<usize>> for Wtf8 {
    406    type Output = Wtf8;
    407
    408    #[inline]
    409    fn index(&self, range: ops::RangeTo<usize>) -> &Wtf8 {
    410        if self.is_code_point_boundary(range.end) {
    411            // SAFETY: is_code_point_boundary checks that the index is valid
    412            unsafe { slice_unchecked(self, 0, range.end) }
    413        } else {
    414            slice_error_fail(self, 0, range.end)
    415        }
    416    }
    417}
    418
    419impl ops::Index<ops::RangeFull> for Wtf8 {
    420    type Output = Wtf8;
    421
    422    #[inline]
    423    fn index(&self, _range: ops::RangeFull) -> &Wtf8 {
    424        self
    425    }
    426}
    427
    428#[inline]
    429fn decode_surrogate(second_byte: u8, third_byte: u8) -> u16 {
    430    // The first byte is assumed to be 0xED
    431    0xD800 | (second_byte as u16 & 0x3F) << 6 | third_byte as u16 & 0x3F
    432}
    433
    434impl Wtf8 {
    435    /// Copied from str::is_char_boundary
    436    #[inline]
    437    pub fn is_code_point_boundary(&self, index: usize) -> bool {
    438        if index == 0 {
    439            return true;
    440        }
    441        match self.bytes.get(index) {
    442            None => index == self.len(),
    443            Some(&b) => (b as i8) >= -0x40,
    444        }
    445    }
    446
    447    /// Verify that `index` is at the edge of either a valid UTF-8 codepoint
    448    /// (i.e. a codepoint that's not a surrogate) or of the whole string.
    449    ///
    450    /// These are the cases currently permitted by `OsStr::self_encoded_bytes`.
    451    /// Splitting between surrogates is valid as far as WTF-8 is concerned, but
    452    /// we do not permit it in the public API because WTF-8 is considered an
    453    /// implementation detail.
    454    #[track_caller]
    455    #[inline]
    456    pub fn check_utf8_boundary(&self, index: usize) {
    457        if index == 0 {
    458            return;
    459        }
    460        match self.bytes.get(index) {
    461            Some(0xED) => (), // Might be a surrogate
    462            Some(&b) if (b as i8) >= -0x40 => return,
    463            Some(_) => panic!("byte index {index} is not a codepoint boundary"),
    464            None if index == self.len() => return,
    465            None => panic!("byte index {index} is out of bounds"),
    466        }
    467        if self.bytes[index + 1] >= 0xA0 {
    468            // There's a surrogate after index. Now check before index.
    469            if index >= 3 && self.bytes[index - 3] == 0xED && self.bytes[index - 2] >= 0xA0 {
    470                panic!("byte index {index} lies between surrogate codepoints");
    471            }
    472        }
    473    }
    474}
    475
    476/// Copied from core::str::raw::slice_unchecked
    477#[inline]
    478unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 {
    479    // SAFETY: memory layout of a &[u8] and &Wtf8 are the same
    480    unsafe {
    481        let len = end - begin;
    482        let start = s.as_bytes().as_ptr().add(begin);
    483        Wtf8::from_bytes_unchecked(slice::from_raw_parts(start, len))
    484    }
    485}
    486
    487/// Copied from core::str::raw::slice_error_fail
    488#[inline(never)]
    489fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! {
    490    assert!(begin <= end);
    491    panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary");
    492}
    493
    494/// Iterator for the code points of a WTF-8 string.
    495///
    496/// Created with the method `.code_points()`.
    497#[derive(Clone)]
    498#[doc(hidden)]
    499pub struct Wtf8CodePoints<'a> {
    500    bytes: slice::Iter<'a, u8>,
    501}
    502
    503impl Iterator for Wtf8CodePoints<'_> {
    504    type Item = CodePoint;
    505
    506    #[inline]
    507    fn next(&mut self) -> Option<CodePoint> {
    508        // SAFETY: `self.bytes` has been created from a WTF-8 string
    509        unsafe { next_code_point(&mut self.bytes).map(|c| CodePoint::from_u32_unchecked(c)) }
    510    }
    511
    512    #[inline]
    513    fn size_hint(&self) -> (usize, Option<usize>) {
    514        let len = self.bytes.len();
    515        (len.saturating_add(3) / 4, Some(len))
    516    }
    517}
    518
    519impl fmt::Debug for Wtf8CodePoints<'_> {
    520    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
    521        f.debug_tuple("Wtf8CodePoints")
    522            // SAFETY: We always leave the string in a valid state after each iteration.
    523            .field(&unsafe { Wtf8::from_bytes_unchecked(self.bytes.as_slice()) })
    524            .finish()
    525    }
    526}
    527
    528/// Generates a wide character sequence for potentially ill-formed UTF-16.
    529#[stable(feature = "rust1", since = "1.0.0")]
    530#[derive(Clone)]
    531#[doc(hidden)]
    532pub struct EncodeWide<'a> {
    533    code_points: Wtf8CodePoints<'a>,
    534    extra: u16,
    535}
    536
    537// Copied from libunicode/u_str.rs
    538#[stable(feature = "rust1", since = "1.0.0")]
    539impl Iterator for EncodeWide<'_> {
    540    type Item = u16;
    541
    542    #[inline]
    543    fn next(&mut self) -> Option<u16> {
    544        if self.extra != 0 {
    545            let tmp = self.extra;
    546            self.extra = 0;
    547            return Some(tmp);
    548        }
    549
    550        let mut buf = [0; char::MAX_LEN_UTF16];
    551        self.code_points.next().map(|code_point| {
    552            let n = encode_utf16_raw(code_point.to_u32(), &mut buf).len();
    553            if n == 2 {
    554                self.extra = buf[1];
    555            }
    556            buf[0]
    557        })
    558    }
    559
    560    #[inline]
    561    fn size_hint(&self) -> (usize, Option<usize>) {
    562        let (low, high) = self.code_points.size_hint();
    563        let ext = (self.extra != 0) as usize;
    564        // every code point gets either one u16 or two u16,
    565        // so this iterator is between 1 or 2 times as
    566        // long as the underlying iterator.
    567        (low + ext, high.and_then(|n| n.checked_mul(2)).and_then(|n| n.checked_add(ext)))
    568    }
    569}
    570
    571#[stable(feature = "encode_wide_fused_iterator", since = "1.62.0")]
    572impl FusedIterator for EncodeWide<'_> {}
    573
    574#[stable(feature = "encode_wide_debug", since = "1.92.0")]
    575impl fmt::Debug for EncodeWide<'_> {
    576    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
    577        struct CodeUnit(u16);
    578        impl fmt::Debug for CodeUnit {
    579            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
    580                // This output attempts to balance readability with precision.
    581                // Render characters which take only one WTF-16 code unit using
    582                // `char` syntax and everything else as code units with hex
    583                // integer syntax (including paired and unpaired surrogate
    584                // halves). Since Rust has no `char`-like type for WTF-16, this
    585                // isn't perfect, so if this output isn't suitable, it is open
    586                // to being changed (see #140153).
    587                match char::from_u32(self.0 as u32) {
    588                    Some(c) => write!(f, "{c:?}"),
    589                    None => write!(f, "0x{:04X}", self.0),
    590                }
    591            }
    592        }
    593
    594        write!(f, "EncodeWide(")?;
    595        f.debug_list().entries(self.clone().map(CodeUnit)).finish()?;
    596        write!(f, ")")?;
    597        Ok(())
    598    }
    599}
    600
    601impl Hash for CodePoint {
    602    #[inline]
    603    fn hash<H: Hasher>(&self, state: &mut H) {
    604        self.0.hash(state)
    605    }
    606}
    607
    608impl Hash for Wtf8 {
    609    #[inline]
    610    fn hash<H: Hasher>(&self, state: &mut H) {
    611        state.write(&self.bytes);
    612        0xfeu8.hash(state)
    613    }
    614}
    615
    616#[unstable(feature = "clone_to_uninit", issue = "126799")]
    617unsafe impl CloneToUninit for Wtf8 {
    618    #[inline]
    619    #[cfg_attr(debug_assertions, track_caller)]
    620    unsafe fn clone_to_uninit(&self, dst: *mut u8) {
    621        // SAFETY: we're just a transparent wrapper around [u8]
    622        unsafe { self.bytes.clone_to_uninit(dst) }
    623    }
    624}