From f0bbc782ac11de6001eeec63e347427dbc639c04 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 26 Apr 2022 13:14:38 +1000
Subject: [PATCH 1/5] Avoid producing `NoDelim` values in `TokenCursorFrame`.
---
compiler/rustc_parse/src/parser/expr.rs | 3 ++-
compiler/rustc_parse/src/parser/mod.rs | 34 ++++++++++---------------
2 files changed, 15 insertions(+), 22 deletions(-)
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 7efc0ca2da234..e4370809ebc24 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -2043,7 +2043,8 @@ impl<'a> Parser<'a> {
self.sess.gated_spans.gate(sym::async_closure, span);
}
- if self.token.kind == TokenKind::Semi && self.token_cursor.frame.delim == DelimToken::Paren
+ if self.token.kind == TokenKind::Semi
+ && matches!(self.token_cursor.frame.delim_sp, Some((DelimToken::Paren, _)))
{
// It is likely that the closure body is a block but where the
// braces have been removed. We will recover and eat the next
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 1686c5873e183..dfe758d0cdf01 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -244,14 +244,13 @@ struct TokenCursor {
#[derive(Clone)]
struct TokenCursorFrame {
- delim: token::DelimToken,
- span: DelimSpan,
+ delim_sp: Option<(DelimToken, DelimSpan)>,
tree_cursor: tokenstream::Cursor,
}
impl TokenCursorFrame {
- fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self {
- TokenCursorFrame { delim, span, tree_cursor: tts.into_trees() }
+ fn new(delim_sp: Option<(DelimToken, DelimSpan)>, tts: TokenStream) -> Self {
+ TokenCursorFrame { delim_sp, tree_cursor: tts.into_trees() }
}
}
@@ -266,7 +265,7 @@ impl TokenCursor {
loop {
// FIXME: we currently don't return `NoDelim` open/close delims. To fix #67062 we will
// need to, whereupon the `delim != DelimToken::NoDelim` conditions below can be
- // removed, as well as the loop.
+ // removed.
if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() {
match tree {
&TokenTree::Token(ref token) => match (desugar_doc_comments, token) {
@@ -277,7 +276,7 @@ impl TokenCursor {
},
&TokenTree::Delimited(sp, delim, ref tts) => {
// Set `open_delim` to true here because we deal with it immediately.
- let frame = TokenCursorFrame::new(sp, delim, tts.clone());
+ let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone());
self.stack.push(mem::replace(&mut self.frame, frame));
if delim != DelimToken::NoDelim {
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
@@ -286,12 +285,11 @@ impl TokenCursor {
}
};
} else if let Some(frame) = self.stack.pop() {
- let delim = self.frame.delim;
- let span = self.frame.span;
- self.frame = frame;
- if delim != DelimToken::NoDelim {
+ if let Some((delim, span)) = self.frame.delim_sp && delim != DelimToken::NoDelim {
+ self.frame = frame;
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
}
+ self.frame = frame;
// No close delimiter to return; continue on to the next iteration.
} else {
return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
@@ -330,8 +328,7 @@ impl TokenCursor {
self.stack.push(mem::replace(
&mut self.frame,
TokenCursorFrame::new(
- delim_span,
- token::NoDelim,
+ None,
if attr_style == AttrStyle::Inner {
[TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body]
.iter()
@@ -431,10 +428,6 @@ impl<'a> Parser<'a> {
desugar_doc_comments: bool,
subparser_name: Option<&'static str>,
) -> Self {
- // Note: because of the way `TokenCursor::inlined_next` is structured, the `span` and
- // `delim` arguments here are never used.
- let start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens);
-
let mut parser = Parser {
sess,
token: Token::dummy(),
@@ -444,7 +437,7 @@ impl<'a> Parser<'a> {
restrictions: Restrictions::empty(),
expected_tokens: Vec::new(),
token_cursor: TokenCursor {
- frame: start_frame,
+ frame: TokenCursorFrame::new(None, tokens),
stack: Vec::new(),
num_next_calls: 0,
desugar_doc_comments,
@@ -1025,7 +1018,7 @@ impl<'a> Parser<'a> {
}
let frame = &self.token_cursor.frame;
- if frame.delim != DelimToken::NoDelim {
+ if let Some((delim, span)) = frame.delim_sp && delim != DelimToken::NoDelim {
let all_normal = (0..dist).all(|i| {
let token = frame.tree_cursor.look_ahead(i);
!matches!(token, Some(TokenTree::Delimited(_, DelimToken::NoDelim, _)))
@@ -1038,7 +1031,7 @@ impl<'a> Parser<'a> {
looker(&Token::new(token::OpenDelim(*delim), dspan.open))
}
},
- None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
+ None => looker(&Token::new(token::CloseDelim(delim), span.close)),
};
}
}
@@ -1198,8 +1191,7 @@ impl<'a> Parser<'a> {
// Grab the tokens from this frame.
let frame = &self.token_cursor.frame;
let stream = frame.tree_cursor.stream.clone();
- let span = frame.span;
- let delim = frame.delim;
+ let (delim, span) = frame.delim_sp.unwrap();
// Advance the token cursor through the entire delimited
// sequence. After getting the `OpenDelim` we are *within* the
From 6b367a05320f2792414281075a4e98452b412a82 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 26 Apr 2022 15:21:15 +1000
Subject: [PATCH 2/5] Avoid producing `NoDelim` values in `MacArgs::delim()`.
---
compiler/rustc_ast/src/ast.rs | 6 +--
compiler/rustc_ast_pretty/src/pprust/state.rs | 38 ++++++++++---------
compiler/rustc_parse/src/parser/stmt.rs | 36 ++++++++++--------
src/tools/rustfmt/src/expr.rs | 2 +-
4 files changed, 44 insertions(+), 38 deletions(-)
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index b7da276fc7e1d..4233f7806248b 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -1542,10 +1542,10 @@ pub enum MacArgs {
}
impl MacArgs {
- pub fn delim(&self) -> DelimToken {
+ pub fn delim(&self) -> Option<DelimToken> {
match self {
- MacArgs::Delimited(_, delim, _) => delim.to_token(),
- MacArgs::Empty | MacArgs::Eq(..) => token::NoDelim,
+ MacArgs::Delimited(_, delim, _) => Some(delim.to_token()),
+ MacArgs::Empty | MacArgs::Eq(..) => None,
}
}
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 3c9bb81bedb1c..39824095e8663 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -464,7 +464,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
Some(MacHeader::Path(&item.path)),
false,
None,
- delim.to_token(),
+ Some(delim.to_token()),
tokens,
true,
span,
@@ -530,7 +530,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
None,
false,
None,
- *delim,
+ Some(*delim),
tts,
convert_dollar_crate,
dspan.entire(),
@@ -556,12 +556,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
header: Option<MacHeader<'_>>,
has_bang: bool,
ident: Option<Ident>,
- delim: DelimToken,
+ delim: Option<DelimToken>,
tts: &TokenStream,
convert_dollar_crate: bool,
span: Span,
) {
- if delim == DelimToken::Brace {
+ if delim == Some(DelimToken::Brace) {
self.cbox(INDENT_UNIT);
}
match header {
@@ -577,7 +577,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
self.print_ident(ident);
}
match delim {
- DelimToken::Brace => {
+ Some(DelimToken::Brace) => {
if header.is_some() || has_bang || ident.is_some() {
self.nbsp();
}
@@ -585,23 +585,25 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
if !tts.is_empty() {
self.space();
}
- }
- _ => {
- let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
- self.word(token_str)
- }
- }
- self.ibox(0);
- self.print_tts(tts, convert_dollar_crate);
- self.end();
- match delim {
- DelimToken::Brace => {
+ self.ibox(0);
+ self.print_tts(tts, convert_dollar_crate);
+ self.end();
let empty = tts.is_empty();
self.bclose(span, empty);
}
- _ => {
+ Some(delim) => {
+ let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
+ self.word(token_str);
+ self.ibox(0);
+ self.print_tts(tts, convert_dollar_crate);
+ self.end();
let token_str = self.token_kind_to_string(&token::CloseDelim(delim));
- self.word(token_str)
+ self.word(token_str);
+ }
+ None => {
+ self.ibox(0);
+ self.print_tts(tts, convert_dollar_crate);
+ self.end();
}
}
}
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 5b7ae5f7a7b82..14f1208b71f51 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -164,25 +164,29 @@ impl<'a> Parser<'a> {
let delim = args.delim();
let hi = self.prev_token.span;
- let style =
- if delim == token::Brace { MacStmtStyle::Braces } else { MacStmtStyle::NoBraces };
+ let style = match delim {
+ Some(token::Brace) => MacStmtStyle::Braces,
+ Some(_) => MacStmtStyle::NoBraces,
+ None => unreachable!(),
+ };
let mac = MacCall { path, args, prior_type_ascription: self.last_type_ascription };
- let kind =
- if (delim == token::Brace && self.token != token::Dot && self.token != token::Question)
- || self.token == token::Semi
- || self.token == token::Eof
- {
- StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
- } else {
- // Since none of the above applied, this is an expression statement macro.
- let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new());
- let e = self.maybe_recover_from_bad_qpath(e, true)?;
- let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
- let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
- StmtKind::Expr(e)
- };
+ let kind = if (style == MacStmtStyle::Braces
+ && self.token != token::Dot
+ && self.token != token::Question)
+ || self.token == token::Semi
+ || self.token == token::Eof
+ {
+ StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
+ } else {
+ // Since none of the above applied, this is an expression statement macro.
+ let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new());
+ let e = self.maybe_recover_from_bad_qpath(e, true)?;
+ let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
+ let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
+ StmtKind::Expr(e)
+ };
Ok(self.mk_stmt(lo.to(hi), kind))
}
diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs
index 4f333cd27cefe..cd724373f4d16 100644
--- a/src/tools/rustfmt/src/expr.rs
+++ b/src/tools/rustfmt/src/expr.rs
@@ -1325,7 +1325,7 @@ pub(crate) fn can_be_overflowed_expr(
}
ast::ExprKind::MacCall(ref mac) => {
match (
- rustc_ast::ast::MacDelimiter::from_token(mac.args.delim()),
+ rustc_ast::ast::MacDelimiter::from_token(mac.args.delim().unwrap()),
context.config.overflow_delimited_expr(),
) {
(Some(ast::MacDelimiter::Bracket), true)
From a8e862cb7d4fd3ce4e71ca27cdef84101dfc1871 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 26 Apr 2022 15:38:10 +1000
Subject: [PATCH 3/5] Avoid producing `NoDelim` values in `Frame`.
The code currently ignores the actual delimiter on the RHS and fakes up
a `NoDelim`/`DelimSpan::dummy()` one. This commit changes it to use the
actual delimiter.
The commit also reorders the fields for the `Delimited` variant to match
the `Sequence` variant.
---
compiler/rustc_expand/src/mbe/macro_rules.rs | 9 ++++-----
compiler/rustc_expand/src/mbe/transcribe.rs | 13 +++++++------
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 2cfd6968accf7..f3dcdbf93c5f1 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -260,16 +260,15 @@ fn generic_extension<'cx, 'tt>(
// Merge the gated spans from parsing the matcher with the pre-existing ones.
sess.gated_spans.merge(gated_spans_snapshot);
- // Ignore the delimiters on the RHS.
- let rhs = match &rhses[i] {
- mbe::TokenTree::Delimited(_, delimited) => &delimited.tts,
+ let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] {
+ mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span),
_ => cx.span_bug(sp, "malformed macro rhs"),
};
let arm_span = rhses[i].span();
- let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>();
+ let rhs_spans = rhs.tts.iter().map(|t| t.span()).collect::<Vec<_>>();
// rhs has holes ( `$id` and `$(...)` that need filled)
- let mut tts = match transcribe(cx, &named_matches, &rhs, transparency) {
+ let mut tts = match transcribe(cx, &named_matches, &rhs, rhs_span, transparency) {
Ok(tts) => tts,
Err(mut err) => {
err.emit();
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index d25f044234cf4..d5b1913e1440c 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -29,8 +29,8 @@ impl MutVisitor for Marker {
enum Frame<'a> {
Delimited {
tts: &'a [mbe::TokenTree],
- delim_token: token::DelimToken,
idx: usize,
+ delim_token: token::DelimToken,
span: DelimSpan,
},
Sequence {
@@ -42,8 +42,8 @@ enum Frame<'a> {
impl<'a> Frame<'a> {
/// Construct a new frame around the delimited set of tokens.
- fn new(tts: &'a [mbe::TokenTree]) -> Frame<'a> {
- Frame::Delimited { tts, delim_token: token::NoDelim, idx: 0, span: DelimSpan::dummy() }
+ fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> {
+ Frame::Delimited { tts: &src.tts, idx: 0, delim_token: src.delim, span }
}
}
@@ -85,17 +85,18 @@ impl<'a> Iterator for Frame<'a> {
pub(super) fn transcribe<'a>(
cx: &ExtCtxt<'a>,
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
- src: &[mbe::TokenTree],
+ src: &mbe::Delimited,
+ src_span: DelimSpan,
transparency: Transparency,
) -> PResult<'a, TokenStream> {
// Nothing for us to transcribe...
- if src.is_empty() {
+ if src.tts.is_empty() {
return Ok(TokenStream::default());
}
// We descend into the RHS (`src`), expanding things as we go. This stack contains the things
// we have yet to expand/are still expanding. We start the stack off with the whole RHS.
- let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src)];
+ let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src, src_span)];
// As we descend in the RHS, we will need to be able to match nested sequences of matchers.
// `repeats` keeps track of where we are in matching at each level, with the last element being
From 9665da35cc493458e94d0b8a8931bf3d785071dc Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 26 Apr 2022 16:21:55 +1000
Subject: [PATCH 4/5] Avoid producing `NoDelim` values in `FrameData`.
---
.../rustc_parse/src/parser/attr_wrapper.rs | 27 ++++++++++---------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
index 02749088c3139..575b01180df77 100644
--- a/compiler/rustc_parse/src/parser/attr_wrapper.rs
+++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -5,7 +5,7 @@ use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream,
use rustc_ast::{self as ast};
use rustc_ast::{AstLike, AttrVec, Attribute};
use rustc_errors::PResult;
-use rustc_span::{sym, Span, DUMMY_SP};
+use rustc_span::{sym, Span};
use std::convert::TryInto;
use std::ops::Range;
@@ -400,24 +400,26 @@ fn make_token_stream(
) -> AttrAnnotatedTokenStream {
#[derive(Debug)]
struct FrameData {
- open: Span,
- open_delim: DelimToken,
+ // This is `None` for the first frame, `Some` for all others.
+ open_delim_sp: Option<(DelimToken, Span)>,
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
}
- let mut stack =
- vec![FrameData { open: DUMMY_SP, open_delim: DelimToken::NoDelim, inner: vec![] }];
+ let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
let mut token_and_spacing = iter.next();
while let Some((token, spacing)) = token_and_spacing {
match token {
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
- stack.push(FrameData { open: span, open_delim: delim, inner: vec![] });
+ stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
}
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
// HACK: If we encounter a mismatched `None` delimiter at the top
// level, just ignore it.
if matches!(delim, DelimToken::NoDelim)
&& (stack.len() == 1
- || !matches!(stack.last_mut().unwrap().open_delim, DelimToken::NoDelim))
+ || !matches!(
+ stack.last_mut().unwrap().open_delim_sp.unwrap().0,
+ DelimToken::NoDelim
+ ))
{
token_and_spacing = iter.next();
continue;
@@ -430,7 +432,7 @@ fn make_token_stream(
// merge our current frame with the one above it. That is, transform
// `[ { < first second } third ]` into `[ { first second } third ]`
if !matches!(delim, DelimToken::NoDelim)
- && matches!(frame_data.open_delim, DelimToken::NoDelim)
+ && matches!(frame_data.open_delim_sp.unwrap().0, DelimToken::NoDelim)
{
stack.last_mut().unwrap().inner.extend(frame_data.inner);
// Process our closing delimiter again, this time at the previous
@@ -439,12 +441,13 @@ fn make_token_stream(
continue;
}
+ let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap();
assert_eq!(
- frame_data.open_delim, delim,
+ open_delim, delim,
"Mismatched open/close delims: open={:?} close={:?}",
- frame_data.open, span
+ open_delim, span
);
- let dspan = DelimSpan::from_pair(frame_data.open, span);
+ let dspan = DelimSpan::from_pair(open_sp, span);
let stream = AttrAnnotatedTokenStream::new(frame_data.inner);
let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream);
stack
@@ -472,7 +475,7 @@ fn make_token_stream(
// HACK: If we don't have a closing `None` delimiter for our last
// frame, merge the frame with the top-level frame. That is,
// turn `< first second` into `first second`
- if stack.len() == 2 && stack[1].open_delim == DelimToken::NoDelim {
+ if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == DelimToken::NoDelim {
let temp_buf = stack.pop().unwrap();
stack.last_mut().unwrap().inner.extend(temp_buf.inner);
}
From 86f011704c31f4eb1e7993e1951826bca46fe8f1 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 26 Apr 2022 15:09:11 +1000
Subject: [PATCH 5/5] Make explicit an unreachable `NoDelim` case in `rustfmt`.
---
src/tools/rustfmt/src/macros.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs
index 664f152e8be1d..92606902c5789 100644
--- a/src/tools/rustfmt/src/macros.rs
+++ b/src/tools/rustfmt/src/macros.rs
@@ -562,7 +562,7 @@ fn delim_token_to_str(
("{ ", " }")
}
}
- DelimToken::NoDelim => ("", ""),
+ DelimToken::NoDelim => unreachable!(),
};
if use_multiple_lines {
let indent_str = shape.indent.to_string_with_newline(context.config);