lossy.rs - source
Description: Source of the Rust file `library/core/src/str/lossy.rs`.

#
core/str/
lossy.rs

    1use super::char::EscapeDebugExtArgs;
    2use super::from_utf8_unchecked;
    3use super::validations::utf8_char_width;
    4use crate::fmt;
    5use crate::fmt::{Formatter, Write};
    6use crate::iter::FusedIterator;
    7
    8impl [u8] {
    9    /// Creates an iterator over the contiguous valid UTF-8 ranges of this
    10    /// slice, and the non-UTF-8 fragments in between.
    11    ///
    12    /// See the [`Utf8Chunk`] type for documentation of the items yielded by this iterator.
    13    ///
    14    /// # Examples
    15    ///
    16    /// This function formats arbitrary but mostly-UTF-8 bytes into Rust source
    17    /// code in the form of a C-string literal (`c"..."`).
    18    ///
    19    /// ```
    20    /// use std::fmt::Write as _;
    21    ///
    22    /// pub fn cstr_literal(bytes: &[u8]) -> String {
    23    ///     let mut repr = String::new();
    24    ///     repr.push_str("c\"");
    25    ///     for chunk in bytes.utf8_chunks() {
    26    ///         for ch in chunk.valid().chars() {
    27    ///             // Escapes \0, \t, \r, \n, \\, \', \", and uses \u{...} for non-printable characters.
    28    ///             write!(repr, "{}", ch.escape_debug()).unwrap();
    29    ///         }
    30    ///         for byte in chunk.invalid() {
    31    ///             write!(repr, "\\x{:02X}", byte).unwrap();
    32    ///         }
    33    ///     }
    34    ///     repr.push('"');
    35    ///     repr
    36    /// }
    37    ///
    38    /// fn main() {
    39    ///     let lit = cstr_literal(b"\xferris the \xf0\x9f\xa6\x80\x07");
    40    ///     let expected = stringify!(c"\xFErris the 🦀\u{7}");
    41    ///     assert_eq!(lit, expected);
    42    /// }
    43    /// ```
    44    #[stable(feature = "utf8_chunks", since = "1.79.0")]
    45    pub fn utf8_chunks(&self) -> Utf8Chunks<'_> {
    46        Utf8Chunks { source: self }
    47    }
    48}
    49
    50/// An item returned by the [`Utf8Chunks`] iterator.
    51///
    52/// A `Utf8Chunk` stores a sequence of [`u8`] up to the first broken character
    53/// when decoding a UTF-8 string.
    54///
    55/// # Examples
    56///
    57/// ```
    58/// // An invalid UTF-8 string
    59/// let bytes = b"foo\xF1\x80bar";
    60///
    61/// // Decode the first `Utf8Chunk`
    62/// let chunk = bytes.utf8_chunks().next().unwrap();
    63///
    64/// // The first three characters are valid UTF-8
    65/// assert_eq!("foo", chunk.valid());
    66///
    67/// // The fourth character is broken
    68/// assert_eq!(b"\xF1\x80", chunk.invalid());
    69/// ```
    70#[stable(feature = "utf8_chunks", since = "1.79.0")]
    71#[derive(Clone, Debug, PartialEq, Eq)]
    72pub struct Utf8Chunk<'a> {
    73    valid: &'a str,
    74    invalid: &'a [u8],
    75}
    76
    77impl<'a> Utf8Chunk<'a> {
    78    /// Returns the next validated UTF-8 substring.
    79    ///
    80    /// This substring can be empty at the start of the string or between
    81    /// broken UTF-8 characters.
    82    #[must_use]
    83    #[stable(feature = "utf8_chunks", since = "1.79.0")]
    84    pub fn valid(&self) -> &'a str {
    85        self.valid
    86    }
    87
    88    /// Returns the invalid sequence that caused a failure.
    89    ///
    90    /// The returned slice will have a maximum length of 3 and starts after the
    91    /// substring given by [`valid`]. Decoding will resume after this sequence.
    92    ///
    93    /// If empty, this is the last chunk in the string. If non-empty, an
    94    /// unexpected byte was encountered or the end of the input was reached
    95    /// unexpectedly.
    96    ///
    97    /// Lossy decoding would replace this sequence with [`U+FFFD REPLACEMENT
    98    /// CHARACTER`].
    99    ///
    100    /// [`valid`]: Self::valid
    101    /// [`U+FFFD REPLACEMENT CHARACTER`]: crate::char::REPLACEMENT_CHARACTER
    102    #[must_use]
    103    #[stable(feature = "utf8_chunks", since = "1.79.0")]
    104    pub fn invalid(&self) -> &'a [u8] {
    105        self.invalid
    106    }
    107}
    108
    109#[must_use]
    110#[unstable(feature = "str_internals", issue = "none")]
    111pub struct Debug<'a>(&'a [u8]);
    112
    113#[unstable(feature = "str_internals", issue = "none")]
    114impl fmt::Debug for Debug<'_> {
    115    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
    116        f.write_char('"')?;
    117
    118        for chunk in self.0.utf8_chunks() {
    119            // Valid part.
    120            // Here we partially parse UTF-8 again which is suboptimal.
    121            {
    122                let valid = chunk.valid();
    123                let mut from = 0;
    124                for (i, c) in valid.char_indices() {
    125                    let esc = c.escape_debug_ext(EscapeDebugExtArgs {
    126                        escape_grapheme_extended: true,
    127                        escape_single_quote: false,
    128                        escape_double_quote: true,
    129                    });
    130                    // If char needs escaping, flush backlog so far and write, else skip
    131                    if esc.len() != 1 {
    132                        f.write_str(&valid[from..i])?;
    133                        for c in esc {
    134                            f.write_char(c)?;
    135                        }
    136                        from = i + c.len_utf8();
    137                    }
    138                }
    139                f.write_str(&valid[from..])?;
    140            }
    141
    142            // Broken parts of string as hex escape.
    143            for &b in chunk.invalid() {
    144                write!(f, "\\x{:02X}", b)?;
    145            }
    146        }
    147
    148        f.write_char('"')
    149    }
    150}
    151
    152/// An iterator used to decode a slice of mostly UTF-8 bytes to string slices
    153/// ([`&str`]) and byte slices ([`&[u8]`][byteslice]).
    154///
    155/// This struct is created by the [`utf8_chunks`] method on bytes slices.
    156/// If you want a simple conversion from UTF-8 byte slices to string slices,
    157/// [`from_utf8`] is easier to use.
    158///
    159/// See the [`Utf8Chunk`] type for documentation of the items yielded by this iterator.
    160///
    161/// [byteslice]: slice
    162/// [`utf8_chunks`]: slice::utf8_chunks
    163/// [`from_utf8`]: super::from_utf8
    164///
    165/// # Examples
    166///
    167/// This can be used to create functionality similar to
    168/// [`String::from_utf8_lossy`] without allocating heap memory:
    169///
    170/// ```
    171/// fn from_utf8_lossy<F>(input: &[u8], mut push: F) where F: FnMut(&str) {
    172///     for chunk in input.utf8_chunks() {
    173///         push(chunk.valid());
    174///
    175///         if !chunk.invalid().is_empty() {
    176///             push("\u{FFFD}");
    177///         }
    178///     }
    179/// }
    180/// ```
    181///
    182/// [`String::from_utf8_lossy`]: ../../std/string/struct.String.html#method.from_utf8_lossy
    183#[must_use = "iterators are lazy and do nothing unless consumed"]
    184#[stable(feature = "utf8_chunks", since = "1.79.0")]
    185#[derive(Clone)]
    186pub struct Utf8Chunks<'a> {
    187    source: &'a [u8],
    188}
    189
    190impl<'a> Utf8Chunks<'a> {
    191    #[doc(hidden)]
    192    #[unstable(feature = "str_internals", issue = "none")]
    193    pub fn debug(&self) -> Debug<'_> {
    194        Debug(self.source)
    195    }
    196}
    197
    198#[stable(feature = "utf8_chunks", since = "1.79.0")]
    199impl<'a> Iterator for Utf8Chunks<'a> {
    200    type Item = Utf8Chunk<'a>;
    201
    202    fn next(&mut self) -> Option<Utf8Chunk<'a>> {
    203        if self.source.is_empty() {
    204            return None;
    205        }
    206
    207        const TAG_CONT_U8: u8 = 128;
    208        fn safe_get(xs: &[u8], i: usize) -> u8 {
    209            *xs.get(i).unwrap_or(&0)
    210        }
    211
    212        let mut i = 0;
    213        let mut valid_up_to = 0;
    214        while i < self.source.len() {
    215            // SAFETY: `i < self.source.len()` per previous line.
    216            // For some reason the following are both significantly slower:
    217            // while let Some(&byte) = self.source.get(i) {
    218            // while let Some(byte) = self.source.get(i).copied() {
    219            let byte = unsafe { *self.source.get_unchecked(i) };
    220            i += 1;
    221
    222            if byte < 128 {
    223                // This could be a `1 => ...` case in the match below, but for
    224                // the common case of all-ASCII inputs, we bypass loading the
    225                // sizeable UTF8_CHAR_WIDTH table into cache.
    226            } else {
    227                let w = utf8_char_width(byte);
    228
    229                match w {
    230                    2 => {
    231                        if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
    232                            break;
    233                        }
    234                        i += 1;
    235                    }
    236                    3 => {
    237                        match (byte, safe_get(self.source, i)) {
    238                            (0xE0, 0xA0..=0xBF) => (),
    239                            (0xE1..=0xEC, 0x80..=0xBF) => (),
    240                            (0xED, 0x80..=0x9F) => (),
    241                            (0xEE..=0xEF, 0x80..=0xBF) => (),
    242                            _ => break,
    243                        }
    244                        i += 1;
    245                        if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
    246                            break;
    247                        }
    248                        i += 1;
    249                    }
    250                    4 => {
    251                        match (byte, safe_get(self.source, i)) {
    252                            (0xF0, 0x90..=0xBF) => (),
    253                            (0xF1..=0xF3, 0x80..=0xBF) => (),
    254                            (0xF4, 0x80..=0x8F) => (),
    255                            _ => break,
    256                        }
    257                        i += 1;
    258                        if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
    259                            break;
    260                        }
    261                        i += 1;
    262                        if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
    263                            break;
    264                        }
    265                        i += 1;
    266                    }
    267                    _ => break,
    268                }
    269            }
    270
    271            valid_up_to = i;
    272        }
    273
    274        // SAFETY: `i <= self.source.len()` because it is only ever incremented
    275        // via `i += 1` and in between every single one of those increments, `i`
    276        // is compared against `self.source.len()`. That happens either
    277        // literally by `i < self.source.len()` in the while-loop's condition,
    278        // or indirectly by `safe_get(self.source, i) & 192 != TAG_CONT_U8`. The
    279        // loop is terminated as soon as the latest `i += 1` has made `i` no
    280        // longer less than `self.source.len()`, which means it'll be at most
    281        // equal to `self.source.len()`.
    282        let (inspected, remaining) = unsafe { self.source.split_at_unchecked(i) };
    283        self.source = remaining;
    284
    285        // SAFETY: `valid_up_to <= i` because it is only ever assigned via
    286        // `valid_up_to = i` and `i` only increases.
    287        let (valid, invalid) = unsafe { inspected.split_at_unchecked(valid_up_to) };
    288
    289        Some(Utf8Chunk {
    290            // SAFETY: All bytes up to `valid_up_to` are valid UTF-8.
    291            valid: unsafe { from_utf8_unchecked(valid) },
    292            invalid,
    293        })
    294    }
    295}
    296
    297#[stable(feature = "utf8_chunks", since = "1.79.0")]
    298impl FusedIterator for Utf8Chunks<'_> {}
    299
    300#[stable(feature = "utf8_chunks", since = "1.79.0")]
    301impl fmt::Debug for Utf8Chunks<'_> {
    302    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
    303        f.debug_struct("Utf8Chunks").field("source", &self.debug()).finish()
    304    }
    305}