Merge #10995

10995: internal: switch from trait-based TokenSource to simple struct of arrays r=matklad a=matklad

cc #10765 

The idea here is to try to simplify the interface as best as we can. The original trait-based approach is a bit over-engineered and hard to debug. Here, we replace callback with just data. The next PR in series will replace the output `TreeSink` trait with data as well. 


The biggest drawback here is that we now require to materialize all parser's input up-front. This is a bad fit for macro by example: when you parse `$e:expr`, you might consume only part of the input. However, today's trait-based solution doesn't really help -- we were already materializing the whole thing! So, let's keep it simple!

Co-authored-by: Aleksey Kladov <[email protected]>

Author: bors[bot]

crates/mbe/src/lib.rs

@@ -10,7 +10,7 @@ mod parser;
 mod expander;
 mod syntax_bridge;
 mod tt_iter;
-mod subtree_source;
+mod to_parser_tokens;
 
 #[cfg(test)]
 mod benchmark;

crates/mbe/src/subtree_source.rs

@@ -12,7 +12,7 @@ use syntax::{
 use tt::buffer::{Cursor, TokenBuffer};
 
 use crate::{
-    subtree_source::SubtreeTokenSource, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
+    to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
 };
 
 /// Convert the syntax node to a `TokenTree` (what macro
@@ -56,9 +56,9 @@ pub fn token_tree_to_syntax_node(
         }
         _ => TokenBuffer::from_subtree(tt),
     };
-    let mut token_source = SubtreeTokenSource::new(&buffer);
+    let parser_tokens = to_parser_tokens(&buffer);
     let mut tree_sink = TtTreeSink::new(buffer.begin());
-    parser::parse(&mut token_source, &mut tree_sink, entry_point);
+    parser::parse(&parser_tokens, &mut tree_sink, entry_point);
     if tree_sink.roots.len() != 1 {
         return Err(ExpandError::ConversionError);
     }

crates/mbe/src/syntax_bridge.rs

@@ -0,0 +1,99 @@
+//! Convert macro-by-example tokens which are specific to macro expansion into a
+//! format that works for our parser.
+
+use syntax::{lex_single_syntax_kind, SyntaxKind, SyntaxKind::*, T};
+use tt::buffer::TokenBuffer;
+
+pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
+    let mut res = parser::Tokens::default();
+
+    let mut current = buffer.begin();
+
+    while !current.eof() {
+        let cursor = current;
+        let tt = cursor.token_tree();
+
+        // Check if it is lifetime
+        if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
+            if punct.char == '\'' {
+                let next = cursor.bump();
+                match next.token_tree() {
+                    Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(_ident), _)) => {
+                        res.push(LIFETIME_IDENT);
+                        current = next.bump();
+                        continue;
+                    }
+                    _ => panic!("Next token must be ident : {:#?}", next.token_tree()),
+                }
+            }
+        }
+
+        current = match tt {
+            Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
+                match leaf {
+                    tt::Leaf::Literal(lit) => {
+                        let is_negated = lit.text.starts_with('-');
+                        let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
+
+                        let kind = lex_single_syntax_kind(inner_text)
+                            .map(|(kind, _error)| kind)
+                            .filter(|kind| {
+                                kind.is_literal()
+                                    && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
+                            })
+                            .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
+
+                        res.push(kind);
+                    }
+                    tt::Leaf::Ident(ident) => match ident.text.as_ref() {
+                        "_" => res.push(T![_]),
+                        i if i.starts_with('\'') => res.push(LIFETIME_IDENT),
+                        _ => match SyntaxKind::from_keyword(&ident.text) {
+                            Some(kind) => res.push(kind),
+                            None => {
+                                let contextual_keyword =
+                                    SyntaxKind::from_contextual_keyword(&ident.text)
+                                        .unwrap_or(SyntaxKind::IDENT);
+                                res.push_ident(contextual_keyword);
+                            }
+                        },
+                    },
+                    tt::Leaf::Punct(punct) => {
+                        let kind = SyntaxKind::from_char(punct.char)
+                            .unwrap_or_else(|| panic!("{:#?} is not a valid punct", punct));
+                        res.push(kind);
+                        if punct.spacing == tt::Spacing::Joint {
+                            res.was_joint();
+                        }
+                    }
+                }
+                cursor.bump()
+            }
+            Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
+                if let Some(d) = subtree.delimiter_kind() {
+                    res.push(match d {
+                        tt::DelimiterKind::Parenthesis => T!['('],
+                        tt::DelimiterKind::Brace => T!['{'],
+                        tt::DelimiterKind::Bracket => T!['['],
+                    });
+                }
+                cursor.subtree().unwrap()
+            }
+            None => match cursor.end() {
+                Some(subtree) => {
+                    if let Some(d) = subtree.delimiter_kind() {
+                        res.push(match d {
+                            tt::DelimiterKind::Parenthesis => T![')'],
+                            tt::DelimiterKind::Brace => T!['}'],
+                            tt::DelimiterKind::Bracket => T![']'],
+                        })
+                    }
+                    cursor.bump()
+                }
+                None => continue,
+            },
+        };
+    }
+
+    res
+}

crates/mbe/src/to_parser_tokens.rs

@@ -1,7 +1,7 @@
 //! A "Parser" structure for token trees. We use this when parsing a declarative
 //! macro definition into a list of patterns and templates.
 
-use crate::{subtree_source::SubtreeTokenSource, ExpandError, ExpandResult, ParserEntryPoint};
+use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
 
 use parser::TreeSink;
 use syntax::SyntaxKind;
@@ -116,10 +116,10 @@ impl<'a> TtIter<'a> {
         }
 
         let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
-        let mut src = SubtreeTokenSource::new(&buffer);
+        let parser_tokens = to_parser_tokens(&buffer);
         let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
 
-        parser::parse(&mut src, &mut sink, entry_point);
+        parser::parse(&parser_tokens, &mut sink, entry_point);
 
         let mut err = if !sink.cursor.is_root() || sink.error {
             Some(err!("expected {:?}", entry_point))

crates/mbe/src/tt_iter.rs

@@ -296,10 +296,7 @@ fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)>
         T![&] => {
             m = p.start();
             p.bump(T![&]);
-            if p.at(IDENT)
-                && p.at_contextual_kw("raw")
-                && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const]))
-            {
+            if p.at_contextual_kw(T![raw]) && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const])) {
                 p.bump_remap(T![raw]);
                 p.bump_any();
             } else {

crates/parser/src/grammar/expressions.rs

@@ -122,14 +122,14 @@ pub(super) fn opt_item(p: &mut Parser, m: Marker) -> Result<(), Marker> {
         has_mods = true;
         abi(p);
     }
-    if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == T![trait] {
+    if p.at_contextual_kw(T![auto]) && p.nth(1) == T![trait] {
         p.bump_remap(T![auto]);
         has_mods = true;
     }
 
     // test default_item
     // default impl T for Foo {}
-    if p.at(IDENT) && p.at_contextual_kw("default") {
+    if p.at_contextual_kw(T![default]) {
         match p.nth(1) {
             T![fn] | T![type] | T![const] | T![impl] => {
                 p.bump_remap(T![default]);
@@ -176,7 +176,7 @@ pub(super) fn opt_item(p: &mut Parser, m: Marker) -> Result<(), Marker> {
 
     // test existential_type
     // existential type Foo: Fn() -> usize;
-    if p.at(IDENT) && p.at_contextual_kw("existential") && p.nth(1) == T![type] {
+    if p.at_contextual_kw(T![existential]) && p.nth(1) == T![type] {
         p.bump_remap(T![existential]);
         has_mods = true;
     }
@@ -224,10 +224,10 @@ fn opt_item_without_modifiers(p: &mut Parser, m: Marker) -> Result<(), Marker> {
         T![type] => type_alias(p, m),
         T![struct] => adt::strukt(p, m),
         T![enum] => adt::enum_(p, m),
-        IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => adt::union(p, m),
+        IDENT if p.at_contextual_kw(T![union]) && p.nth(1) == IDENT => adt::union(p, m),
 
         T![macro] => macro_def(p, m),
-        IDENT if p.at_contextual_kw("macro_rules") && p.nth(1) == BANG => macro_rules(p, m),
+        IDENT if p.at_contextual_kw(T![macro_rules]) && p.nth(1) == BANG => macro_rules(p, m),
 
         T![const] if (la == IDENT || la == T![_] || la == T![mut]) => consts::konst(p, m),
         T![static] => consts::static_(p, m),
@@ -319,7 +319,7 @@ pub(crate) fn extern_item_list(p: &mut Parser) {
 }
 
 fn macro_rules(p: &mut Parser, m: Marker) {
-    assert!(p.at_contextual_kw("macro_rules"));
+    assert!(p.at_contextual_kw(T![macro_rules]));
     p.bump_remap(T![macro_rules]);
     p.expect(T![!]);
 

crates/parser/src/grammar/items.rs

@@ -10,7 +10,7 @@ pub(super) fn strukt(p: &mut Parser, m: Marker) {
 // test union_item
 // struct U { i: i32, f: f32 }
 pub(super) fn union(p: &mut Parser, m: Marker) {
-    assert!(p.at_contextual_kw("union"));
+    assert!(p.at_contextual_kw(T![union]));
     p.bump_remap(T![union]);
     struct_or_union(p, m, false);
 }