From 275bf626f615f7f154249606ad369d6c142801a5 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 26 Sep 2020 18:46:19 +0300 Subject: [PATCH 1/2] pretty-print-reparse hack: Rename some variables for clarity --- compiler/rustc_parse/src/lib.rs | 37 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 21bbdc9ba8d..a7c8eaa4b15 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -299,7 +299,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // FIXME(#43081): Avoid this pretty-print + reparse hack let source = pprust::nonterminal_to_string(nt); let filename = FileName::macro_expansion_source_code(&source); - let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span)); + let reparsed_tokens = parse_stream_from_source_str(filename, source, sess, Some(span)); // During early phases of the compiler the AST could get modified // directly (e.g., attributes added or removed) and the internal cache @@ -325,7 +325,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // modifications, including adding/removing typically non-semantic // tokens such as extra braces and commas, don't happen. if let Some(tokens) = tokens { - if tokenstream_probably_equal_for_proc_macro(&tokens, &tokens_for_real, sess) { + if tokenstream_probably_equal_for_proc_macro(&tokens, &reparsed_tokens, sess) { return tokens; } info!( @@ -333,9 +333,9 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke going with stringified version" ); info!("cached tokens: {:?}", tokens); - info!("reparsed tokens: {:?}", tokens_for_real); + info!("reparsed tokens: {:?}", reparsed_tokens); } - tokens_for_real + reparsed_tokens } // See comments in `Nonterminal::to_tokenstream` for why we care about @@ -344,8 +344,8 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // This is otherwise the same as `eq_unspanned`, only recursing with a // different method. pub fn tokenstream_probably_equal_for_proc_macro( - first: &TokenStream, - other: &TokenStream, + tokens: &TokenStream, + reparsed_tokens: &TokenStream, sess: &ParseSess, ) -> bool { // When checking for `probably_eq`, we ignore certain tokens that aren't @@ -460,10 +460,11 @@ pub fn tokenstream_probably_equal_for_proc_macro( // Break tokens after we expand any nonterminals, so that we break tokens // that are produced as a result of nonterminal expansion. - let t1 = first.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens); - let t2 = other.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens); + let tokens = tokens.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens); + let reparsed_tokens = + reparsed_tokens.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens); - t1.eq_by(t2, |t1, t2| tokentree_probably_equal_for_proc_macro(&t1, &t2, sess)) + tokens.eq_by(reparsed_tokens, |t, rt| tokentree_probably_equal_for_proc_macro(&t, &rt, sess)) } // See comments in `Nonterminal::to_tokenstream` for why we care about @@ -472,16 +473,20 @@ pub fn tokenstream_probably_equal_for_proc_macro( // This is otherwise the same as `eq_unspanned`, only recursing with a // different method. pub fn tokentree_probably_equal_for_proc_macro( - first: &TokenTree, - other: &TokenTree, + token: &TokenTree, + reparsed_token: &TokenTree, sess: &ParseSess, ) -> bool { - match (first, other) { - (TokenTree::Token(token), TokenTree::Token(token2)) => { - token_probably_equal_for_proc_macro(token, token2) + match (token, reparsed_token) { + (TokenTree::Token(token), TokenTree::Token(reparsed_token)) => { + token_probably_equal_for_proc_macro(token, reparsed_token) } - (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { - delim == delim2 && tokenstream_probably_equal_for_proc_macro(&tts, &tts2, sess) + ( + TokenTree::Delimited(_, delim, tokens), + TokenTree::Delimited(_, reparsed_delim, reparsed_tokens), + ) => { + delim == reparsed_delim + && tokenstream_probably_equal_for_proc_macro(tokens, reparsed_tokens, sess) } _ => false, } From fe3e5aa729ee34749ae730bbb5fd9c906877b82a Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 26 Sep 2020 20:27:14 +0300 Subject: [PATCH 2/2] pretty-print-reparse hack: Remove an impossible case Delimiters cannot appear as isolated tokens in a token stream --- compiler/rustc_parse/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index a7c8eaa4b15..d59dd4016a9 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -7,7 +7,7 @@ #![feature(or_patterns)] use rustc_ast as ast; -use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind}; +use rustc_ast::token::{self, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{self, Spacing, TokenStream, TokenTree}; use rustc_ast_pretty::pprust; use rustc_data_structures::sync::Lrc; @@ -359,9 +359,6 @@ pub fn tokenstream_probably_equal_for_proc_macro( // The pretty printer tends to add trailing commas to // everything, and in particular, after struct fields. | token::Comma - // The pretty printer emits `NoDelim` as whitespace. - | token::OpenDelim(DelimToken::NoDelim) - | token::CloseDelim(DelimToken::NoDelim) // The pretty printer collapses many semicolons into one. | token::Semi // We don't preserve leading `|` tokens in patterns, so