quote.rs - source
Description: Source of the Rust file `library/proc_macro/src/quote.rs`.

#
proc_macro/
quote.rs

    1//! # Quasiquoter
    2//! This file contains the implementation internals of the quasiquoter provided by `quote!`.
    3
    4//! This quasiquoter uses macros 2.0 hygiene to reliably access
    5//! items from `proc_macro`, to build a `proc_macro::TokenStream`.
    6
    7use crate::{
    8    BitOr, Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
    9};
    10
    11#[doc(hidden)]
    12pub struct HasIterator; // True
    13#[doc(hidden)]
    14pub struct ThereIsNoIteratorInRepetition; // False
    15
    16impl BitOr<ThereIsNoIteratorInRepetition> for ThereIsNoIteratorInRepetition {
    17    type Output = ThereIsNoIteratorInRepetition;
    18    fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> ThereIsNoIteratorInRepetition {
    19        ThereIsNoIteratorInRepetition
    20    }
    21}
    22
    23impl BitOr<ThereIsNoIteratorInRepetition> for HasIterator {
    24    type Output = HasIterator;
    25    fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> HasIterator {
    26        HasIterator
    27    }
    28}
    29
    30impl BitOr<HasIterator> for ThereIsNoIteratorInRepetition {
    31    type Output = HasIterator;
    32    fn bitor(self, _rhs: HasIterator) -> HasIterator {
    33        HasIterator
    34    }
    35}
    36
    37impl BitOr<HasIterator> for HasIterator {
    38    type Output = HasIterator;
    39    fn bitor(self, _rhs: HasIterator) -> HasIterator {
    40        HasIterator
    41    }
    42}
    43
    44/// Extension traits used by the implementation of `quote!`. These are defined
    45/// in separate traits, rather than as a single trait due to ambiguity issues.
    46///
    47/// These traits expose a `quote_into_iter` method which should allow calling
    48/// whichever impl happens to be applicable. Calling that method repeatedly on
    49/// the returned value should be idempotent.
    50#[doc(hidden)]
    51pub mod ext {
    52    use core::slice;
    53    use std::collections::btree_set::{self, BTreeSet};
    54
    55    use super::{
    56        HasIterator as HasIter, RepInterp, ThereIsNoIteratorInRepetition as DoesNotHaveIter,
    57    };
    58    use crate::ToTokens;
    59
    60    /// Extension trait providing the `quote_into_iter` method on iterators.
    61    #[doc(hidden)]
    62    pub trait RepIteratorExt: Iterator + Sized {
    63        fn quote_into_iter(self) -> (Self, HasIter) {
    64            (self, HasIter)
    65        }
    66    }
    67
    68    impl<T: Iterator> RepIteratorExt for T {}
    69
    70    /// Extension trait providing the `quote_into_iter` method for
    71    /// non-iterable types. These types interpolate the same value in each
    72    /// iteration of the repetition.
    73    #[doc(hidden)]
    74    pub trait RepToTokensExt {
    75        /// Pretend to be an iterator for the purposes of `quote_into_iter`.
    76        /// This allows repeated calls to `quote_into_iter` to continue
    77        /// correctly returning DoesNotHaveIter.
    78        fn next(&self) -> Option<&Self> {
    79            Some(self)
    80        }
    81
    82        fn quote_into_iter(&self) -> (&Self, DoesNotHaveIter) {
    83            (self, DoesNotHaveIter)
    84        }
    85    }
    86
    87    impl<T: ToTokens + ?Sized> RepToTokensExt for T {}
    88
    89    /// Extension trait providing the `quote_into_iter` method for types that
    90    /// can be referenced as an iterator.
    91    #[doc(hidden)]
    92    pub trait RepAsIteratorExt<'q> {
    93        type Iter: Iterator;
    94
    95        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter);
    96    }
    97
    98    impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &T {
    99        type Iter = T::Iter;
    100
    101        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    102            <T as RepAsIteratorExt>::quote_into_iter(*self)
    103        }
    104    }
    105
    106    impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &mut T {
    107        type Iter = T::Iter;
    108
    109        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    110            <T as RepAsIteratorExt>::quote_into_iter(*self)
    111        }
    112    }
    113
    114    impl<'q, T: 'q> RepAsIteratorExt<'q> for [T] {
    115        type Iter = slice::Iter<'q, T>;
    116
    117        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    118            (self.iter(), HasIter)
    119        }
    120    }
    121
    122    impl<'q, T: 'q, const N: usize> RepAsIteratorExt<'q> for [T; N] {
    123        type Iter = slice::Iter<'q, T>;
    124
    125        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    126            (self.iter(), HasIter)
    127        }
    128    }
    129
    130    impl<'q, T: 'q> RepAsIteratorExt<'q> for Vec<T> {
    131        type Iter = slice::Iter<'q, T>;
    132
    133        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    134            (self.iter(), HasIter)
    135        }
    136    }
    137
    138    impl<'q, T: 'q> RepAsIteratorExt<'q> for BTreeSet<T> {
    139        type Iter = btree_set::Iter<'q, T>;
    140
    141        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    142            (self.iter(), HasIter)
    143        }
    144    }
    145
    146    impl<'q, T: RepAsIteratorExt<'q>> RepAsIteratorExt<'q> for RepInterp<T> {
    147        type Iter = T::Iter;
    148
    149        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
    150            self.0.quote_into_iter()
    151        }
    152    }
    153}
    154
    155// Helper type used within interpolations to allow for repeated binding names.
    156// Implements the relevant traits, and exports a dummy `next()` method.
    157#[derive(Copy, Clone)]
    158#[doc(hidden)]
    159pub struct RepInterp<T>(pub T);
    160
    161impl<T> RepInterp<T> {
    162    // This method is intended to look like `Iterator::next`, and is called when
    163    // a name is bound multiple times, as the previous binding will shadow the
    164    // original `Iterator` object. This allows us to avoid advancing the
    165    // iterator multiple times per iteration.
    166    pub fn next(self) -> Option<T> {
    167        Some(self.0)
    168    }
    169}
    170
    171impl<T: Iterator> Iterator for RepInterp<T> {
    172    type Item = T::Item;
    173
    174    fn next(&mut self) -> Option<Self::Item> {
    175        self.0.next()
    176    }
    177}
    178
    179impl<T: ToTokens> ToTokens for RepInterp<T> {
    180    fn to_tokens(&self, tokens: &mut TokenStream) {
    181        self.0.to_tokens(tokens);
    182    }
    183}
    184
    185macro_rules! minimal_quote_tt {
    186    (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) };
    187    ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, minimal_quote!($($t)*)) };
    188    ({$($t:tt)*}) => { Group::new(Delimiter::Brace, minimal_quote!($($t)*)) };
    189    (,) => { Punct::new(',', Spacing::Alone) };
    190    (.) => { Punct::new('.', Spacing::Alone) };
    191    (;) => { Punct::new(';', Spacing::Alone) };
    192    (!) => { Punct::new('!', Spacing::Alone) };
    193    (<) => { Punct::new('<', Spacing::Alone) };
    194    (>) => { Punct::new('>', Spacing::Alone) };
    195    (&) => { Punct::new('&', Spacing::Alone) };
    196    (=) => { Punct::new('=', Spacing::Alone) };
    197    (#) => { Punct::new('#', Spacing::Alone) };
    198    (|) => { Punct::new('|', Spacing::Alone) };
    199    (:) => { Punct::new(':', Spacing::Alone) };
    200    (*) => { Punct::new('*', Spacing::Alone) };
    201    (_) => { Ident::new("_", Span::def_site()) };
    202    ($i:ident) => { Ident::new(stringify!($i), Span::def_site()) };
    203    ($lit:literal) => { stringify!($lit).parse::<Literal>().unwrap() };
    204}
    205
    206macro_rules! minimal_quote_ts {
    207    ((@ $($t:tt)*)) => { $($t)* };
    208    (::) => {
    209        {
    210            let mut c = (
    211                TokenTree::from(Punct::new(':', Spacing::Joint)),
    212                TokenTree::from(Punct::new(':', Spacing::Alone))
    213            );
    214            c.0.set_span(Span::def_site());
    215            c.1.set_span(Span::def_site());
    216            [c.0, c.1].into_iter().collect::<TokenStream>()
    217        }
    218    };
    219    (=>) => {
    220        {
    221            let mut c = (
    222                TokenTree::from(Punct::new('=', Spacing::Joint)),
    223                TokenTree::from(Punct::new('>', Spacing::Alone))
    224            );
    225            c.0.set_span(Span::def_site());
    226            c.1.set_span(Span::def_site());
    227            [c.0, c.1].into_iter().collect::<TokenStream>()
    228        }
    229    };
    230    (+=) => {
    231        {
    232            let mut c = (
    233                TokenTree::from(Punct::new('+', Spacing::Joint)),
    234                TokenTree::from(Punct::new('=', Spacing::Alone))
    235            );
    236            c.0.set_span(Span::def_site());
    237            c.1.set_span(Span::def_site());
    238            [c.0, c.1].into_iter().collect::<TokenStream>()
    239        }
    240    };
    241    (!=) => {
    242        {
    243            let mut c = (
    244                TokenTree::from(Punct::new('!', Spacing::Joint)),
    245                TokenTree::from(Punct::new('=', Spacing::Alone))
    246            );
    247            c.0.set_span(Span::def_site());
    248            c.1.set_span(Span::def_site());
    249            [c.0, c.1].into_iter().collect::<TokenStream>()
    250        }
    251    };
    252    ($t:tt) => { TokenTree::from(minimal_quote_tt!($t)) };
    253}
    254
    255/// Simpler version of the real `quote!` macro, implemented solely
    256/// through `macro_rules`, for bootstrapping the real implementation
    257/// (see the `quote` function), which does not have access to the
    258/// real `quote!` macro due to the `proc_macro` crate not being
    259/// able to depend on itself.
    260///
    261/// Note: supported tokens are a subset of the real `quote!`, but
    262/// unquoting is different: instead of `$x`, this uses `(@ expr)`.
    263macro_rules! minimal_quote {
    264    ($($t:tt)*) => {
    265        {
    266            #[allow(unused_mut)] // In case the expansion is empty
    267            let mut ts = TokenStream::new();
    268            $(ToTokens::to_tokens(&minimal_quote_ts!($t), &mut ts);)*
    269            ts
    270        }
    271    };
    272}
    273
    274/// Quote a `TokenStream` into a `TokenStream`.
    275/// This is the actual implementation of the `quote!()` proc macro.
    276///
    277/// It is loaded by the compiler in `register_builtin_macros`.
    278#[unstable(feature = "proc_macro_quote", issue = "54722")]
    279pub fn quote(stream: TokenStream) -> TokenStream {
    280    if stream.is_empty() {
    281        return minimal_quote!(crate::TokenStream::new());
    282    }
    283    let proc_macro_crate = minimal_quote!(crate);
    284    let mut after_dollar = false;
    285
    286    let mut tokens = crate::TokenStream::new();
    287    let mut iter = stream.into_iter().peekable();
    288    while let Some(tree) = iter.next() {
    289        if after_dollar {
    290            after_dollar = false;
    291            match tree {
    292                TokenTree::Group(tt) => {
    293                    // Handles repetition by expanding `$( CONTENTS ) SEP_OPT *` to `{ REP_EXPANDED }`.
    294                    let contents = tt.stream();
    295
    296                    // The `*` token is also consumed here.
    297                    let sep_opt: Option<Punct> = match (iter.next(), iter.peek()) {
    298                        (Some(TokenTree::Punct(sep)), Some(TokenTree::Punct(star)))
    299                            if sep.spacing() == Spacing::Joint && star.as_char() == '*' =>
    300                        {
    301                            iter.next();
    302                            Some(sep)
    303                        }
    304                        (Some(TokenTree::Punct(star)), _) if star.as_char() == '*' => None,
    305                        _ => panic!("`$(...)` must be followed by `*` in `quote!`"),
    306                    };
    307
    308                    let mut rep_expanded = TokenStream::new();
    309
    310                    // Append setup code for a `while`, where recursively quoted `CONTENTS`
    311                    // and `SEP_OPT` are repeatedly processed, to `REP_EXPANDED`.
    312                    let meta_vars = collect_meta_vars(contents.clone());
    313                    minimal_quote!(
    314                        use crate::ext::*;
    315                        (@ if sep_opt.is_some() {
    316                            minimal_quote!(let mut _i = 0usize;)
    317                        } else {
    318                            minimal_quote!(();)
    319                        })
    320                        let has_iter = crate::ThereIsNoIteratorInRepetition;
    321                    )
    322                    .to_tokens(&mut rep_expanded);
    323                    for meta_var in &meta_vars {
    324                        minimal_quote!(
    325                            #[allow(unused_mut)]
    326                            let (mut (@ meta_var), i) = (@ meta_var).quote_into_iter();
    327                            let has_iter = has_iter | i;
    328                        )
    329                        .to_tokens(&mut rep_expanded);
    330                    }
    331                    minimal_quote!(let _: crate::HasIterator = has_iter;)
    332                        .to_tokens(&mut rep_expanded);
    333
    334                    // Append the `while` to `REP_EXPANDED`.
    335                    let mut while_body = TokenStream::new();
    336                    for meta_var in &meta_vars {
    337                        minimal_quote!(
    338                            let (@ meta_var) = match (@ meta_var).next() {
    339                                Some(_x) => crate::RepInterp(_x),
    340                                None => break,
    341                            };
    342                        )
    343                        .to_tokens(&mut while_body);
    344                    }
    345                    minimal_quote!(
    346                        (@ if let Some(sep) = sep_opt {
    347                            minimal_quote!(
    348                                if _i > 0 {
    349                                    (@ minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
    350                                        (@ TokenTree::from(Literal::character(sep.as_char()))),
    351                                        (@ minimal_quote!(crate::Spacing::Alone)),
    352                                    )), &mut ts);))
    353                                }
    354                                _i += 1;
    355                            )
    356                        } else {
    357                            minimal_quote!(();)
    358                        })
    359                        (@ quote(contents.clone())).to_tokens(&mut ts);
    360                    )
    361                        .to_tokens(&mut while_body);
    362                    rep_expanded.extend(vec![
    363                        TokenTree::Ident(Ident::new("while", Span::call_site())),
    364                        TokenTree::Ident(Ident::new("true", Span::call_site())),
    365                        TokenTree::Group(Group::new(Delimiter::Brace, while_body)),
    366                    ]);
    367
    368                    minimal_quote!((@ TokenTree::Group(Group::new(Delimiter::Brace, rep_expanded)))).to_tokens(&mut tokens);
    369                    continue;
    370                }
    371                TokenTree::Ident(_) => {
    372                    minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);)
    373                        .to_tokens(&mut tokens);
    374                    continue;
    375                }
    376                TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
    377                _ => panic!(
    378                    "`$` must be followed by an ident or `$` or a repetition group in `quote!`"
    379                ),
    380            }
    381        } else if let TokenTree::Punct(ref tt) = tree {
    382            if tt.as_char() == '$' {
    383                after_dollar = true;
    384                continue;
    385            }
    386        }
    387
    388        match tree {
    389            TokenTree::Punct(tt) => {
    390                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
    391                    (@ TokenTree::from(Literal::character(tt.as_char()))),
    392                    (@ match tt.spacing() {
    393                        Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
    394                        Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
    395                    }),
    396                )), &mut ts);)
    397            }
    398            TokenTree::Group(tt) => {
    399                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
    400                    (@ match tt.delimiter() {
    401                        Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
    402                        Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
    403                        Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
    404                        Delimiter::None => minimal_quote!(crate::Delimiter::None),
    405                    }),
    406                    (@ quote(tt.stream())),
    407                )), &mut ts);)
    408            }
    409            TokenTree::Ident(tt) => {
    410                let literal = tt.to_string();
    411                let (literal, ctor) = if let Some(stripped) = literal.strip_prefix("r#") {
    412                    (stripped, minimal_quote!(crate::Ident::new_raw))
    413                } else {
    414                    (literal.as_str(), minimal_quote!(crate::Ident::new))
    415                };
    416                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident((@ ctor)(
    417                    (@ TokenTree::from(Literal::string(literal))),
    418                    (@ quote_span(proc_macro_crate.clone(), tt.span())),
    419                )), &mut ts);)
    420            }
    421            TokenTree::Literal(tt) => {
    422                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
    423                    let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
    424                        .parse::<crate::TokenStream>()
    425                        .unwrap()
    426                        .into_iter();
    427                    if let (Some(crate::TokenTree::Literal(mut lit)), None) =
    428                        (iter.next(), iter.next())
    429                    {
    430                        lit.set_span((@ quote_span(proc_macro_crate.clone(), tt.span())));
    431                        lit
    432                    } else {
    433                        unreachable!()
    434                    }
    435                }), &mut ts);)
    436            }
    437        }
    438        .to_tokens(&mut tokens);
    439    }
    440    if after_dollar {
    441        panic!("unexpected trailing `$` in `quote!`");
    442    }
    443
    444    minimal_quote! {
    445        {
    446            let mut ts = crate::TokenStream::new();
    447            (@ tokens)
    448            ts
    449        }
    450    }
    451}
    452
    453/// Helper function to support macro repetitions like `$( CONTENTS ) SEP_OPT *` in `quote!`.
    454/// Recursively collects all `Ident`s (meta-variables) that follow a `$`
    455/// from the given `CONTENTS` stream, preserving their order of appearance.
    456fn collect_meta_vars(content_stream: TokenStream) -> Vec<Ident> {
    457    fn helper(stream: TokenStream, out: &mut Vec<Ident>) {
    458        let mut iter = stream.into_iter().peekable();
    459        while let Some(tree) = iter.next() {
    460            match &tree {
    461                TokenTree::Punct(tt) if tt.as_char() == '$' => {
    462                    if let Some(TokenTree::Ident(id)) = iter.peek() {
    463                        out.push(id.clone());
    464                        iter.next();
    465                    }
    466                }
    467                TokenTree::Group(tt) => {
    468                    helper(tt.stream(), out);
    469                }
    470                _ => {}
    471            }
    472        }
    473    }
    474
    475    let mut vars = Vec::new();
    476    helper(content_stream, &mut vars);
    477    vars
    478}
    479
    480/// Quote a `Span` into a `TokenStream`.
    481/// This is needed to implement a custom quoter.
    482#[unstable(feature = "proc_macro_quote", issue = "54722")]
    483pub fn quote_span(proc_macro_crate: TokenStream, span: Span) -> TokenStream {
    484    let id = span.save_span();
    485    minimal_quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
    486}