glue tokens when building token stream

Author: matklad

src/libsyntax/parse/lexer/tokentrees.rs

@@ -39,29 +39,29 @@ struct TokenTreesReader<'a> {
 impl<'a> TokenTreesReader<'a> {
     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
     fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
-        let mut tts = Vec::new();
+        let mut buf = TokenStreamBuilder::default();
 
         self.real_token();
         while self.token != token::Eof {
-            tts.push(self.parse_token_tree()?);
+            buf.push(self.parse_token_tree()?);
         }
 
-        Ok(TokenStream::new(tts))
+        Ok(buf.into_token_stream())
     }
 
     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
-        let mut tts = vec![];
+        let mut buf = TokenStreamBuilder::default();
         loop {
             if let token::CloseDelim(..) = self.token.kind {
-                return TokenStream::new(tts);
+                return buf.into_token_stream();
             }
 
             match self.parse_token_tree() {
-                Ok(tree) => tts.push(tree),
+                Ok(tree) => buf.push(tree),
                 Err(mut e) => {
                     e.emit();
-                    return TokenStream::new(tts);
+                    return buf.into_token_stream();
                 }
             }
         }
@@ -223,8 +223,32 @@ impl<'a> TokenTreesReader<'a> {
                 _ => {
                     self.token = token;
                     return;
-                },
+                }
+            }
+        }
+    }
+}
+
+#[derive(Default)]
+struct TokenStreamBuilder {
+    buf: Vec<TreeAndJoint>,
+}
+
+impl TokenStreamBuilder {
+    fn push(&mut self, (tree, joint): TreeAndJoint) {
+        if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
+            if let TokenTree::Token(token) = &tree {
+                if let Some(glued) = prev_token.glue(token) {
+                    self.buf.pop();
+                    self.buf.push((TokenTree::Token(glued), joint));
+                    return;
+                }
             }
         }
+        self.buf.push((tree, joint))
+    }
+
+    fn into_token_stream(self) -> TokenStream {
+        TokenStream::new(self.buf)
     }
 }

src/libsyntax/parse/token.rs

@@ -551,7 +551,7 @@ impl Token {
         }
     }
 
-    crate fn glue(self, joint: Token) -> Option<Token> {
+    crate fn glue(&self, joint: &Token) -> Option<Token> {
         let kind = match self.kind {
             Eq => match joint.kind {
                 Eq => EqEq,

src/libsyntax/tokenstream.rs

@@ -414,7 +414,7 @@ impl TokenStreamBuilder {
         let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
         if let Some(TokenTree::Token(last_token)) = last_tree_if_joint {
             if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() {
-                if let Some(glued_tok) = last_token.glue(token) {
+                if let Some(glued_tok) = last_token.glue(&token) {
                     let last_stream = self.0.pop().unwrap();
                     self.push_all_but_last_tree(&last_stream);
                     let glued_tt = TokenTree::Token(glued_tok);