OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rollup merge of #61026 - estebank:macro-eof-spans, r=petrochenkov

Browse Source 
Tweak macro parse errors when reaching EOF during macro call parse

Add detail on origin of current parser when reaching EOF, stop saying "found `<eof>`" and point at the end of macro calls.

Fix #27569.
This commit is contained in:
Mazdak Farrokhzad 2019-05-25 04:55:28 +02:00 committed by GitHub
commit 519b040b63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 184 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/librustc/traits/on_unimplemented.rs
View File

@ -226,12 +226,12 @@ impl<'a, 'gcx, 'tcx> OnUnimplementedFormatString {
Ok(result)
}
fn verify(&self,
tcx: TyCtxt<'a, 'gcx, 'tcx>,
trait_def_id: DefId,
span: Span)
-> Result<(), ErrorReported>
{
fn verify(
&self,
tcx: TyCtxt<'a, 'gcx, 'tcx>,
trait_def_id: DefId,
span: Span,
) -> Result<(), ErrorReported> {
let name = tcx.item_name(trait_def_id);
let generics = tcx.generics_of(trait_def_id);
let parser = Parser::new(&self.0, None, vec![], false);
@ -272,12 +272,12 @@ impl<'a, 'gcx, 'tcx> OnUnimplementedFormatString {
result
}
pub fn format(&self,
tcx: TyCtxt<'a, 'gcx, 'tcx>,
trait_ref: ty::TraitRef<'tcx>,
options: &FxHashMap<String, String>)
-> String
{
pub fn format(
&self,
tcx: TyCtxt<'a, 'gcx, 'tcx>,
trait_ref: ty::TraitRef<'tcx>,
options: &FxHashMap<String, String>,
) -> String {
let name = tcx.item_name(trait_ref.def_id);
let trait_str = tcx.def_path_str(trait_ref.def_id);
let generics = tcx.generics_of(trait_ref.def_id);

9
src/libsyntax/attr/mod.rs
View File

@ -278,7 +278,14 @@ impl Attribute {
pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T>
where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>,
{
let mut parser = Parser::new(sess, self.tokens.clone(), None, false, false);
let mut parser = Parser::new(
sess,
self.tokens.clone(),
None,
false,
false,
Some("attribute"),
);
let result = f(&mut parser)?;
if parser.token != token::Eof {
parser.unexpected()?;

4
src/libsyntax/ext/base.rs
View File

@ -11,7 +11,7 @@ use crate::parse::{self, parser, DirectoryOwnership};
use crate::parse::token;
use crate::ptr::P;
use crate::symbol::{kw, sym, Ident, Symbol};
use crate::ThinVec;
use crate::{ThinVec, MACRO_ARGUMENTS};
use crate::tokenstream::{self, TokenStream};
use errors::{DiagnosticBuilder, DiagnosticId};
@ -850,7 +850,7 @@ impl<'a> ExtCtxt<'a> {
}
pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> {
parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect())
parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect(), MACRO_ARGUMENTS)
}
pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() }
pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }

9
src/libsyntax/ext/tt/macro_parser.rs
View File

@ -658,7 +658,14 @@ pub fn parse(
recurse_into_modules: bool,
) -> NamedParseResult {
// Create a parser that can be used for the "black box" parts.
let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
let mut parser = Parser::new(
sess,
tts,
directory,
recurse_into_modules,
true,
crate::MACRO_ARGUMENTS,
);
// A queue of possible matcher positions. We initialize it with the matcher position in which
// the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then

2
src/libsyntax/ext/tt/macro_rules.rs
View File

@ -172,7 +172,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>,
path: Cow::from(cx.current_expansion.module.directory.as_path()),
ownership: cx.current_expansion.directory_ownership,
};
let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false);
let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None);
p.root_module_name = cx.current_expansion.module.mod_path.last()
.map(|id| id.as_str().to_string());

2
src/libsyntax/lib.rs
View File

@ -31,6 +31,8 @@ pub use rustc_data_structures::thin_vec::ThinVec;
use ast::AttrId;
use syntax_pos::edition::Edition;
const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments");
// A variant of 'try!' that panics on an Err. This is used as a crutch on the
// way towards a non-panic!-prone parser. It should be used for fatal parsing
// errors; eventually we plan to convert all code using panictry to just use

75
src/libsyntax/parse/diagnostics.rs
View File

@ -13,7 +13,7 @@ use crate::symbol::kw;
use crate::ThinVec;
use errors::{Applicability, DiagnosticBuilder};
use log::debug;
use syntax_pos::Span;
use syntax_pos::{Span, DUMMY_SP};
pub trait RecoverQPath: Sized + 'static {
const PATH_STYLE: PathStyle = PathStyle::Expr;
@ -201,7 +201,7 @@ impl<'a> Parser<'a> {
let mut path = ast::Path {
segments: Vec::new(),
span: syntax_pos::DUMMY_SP,
span: DUMMY_SP,
};
self.parse_path_segments(&mut path.segments, T::PATH_STYLE)?;
path.span = ty_span.to(self.prev_span);
@ -267,6 +267,58 @@ impl<'a> Parser<'a> {
}
}
/// Create a `DiagnosticBuilder` for an unexpected token `t` and try to recover if it is a
/// closing delimiter.
pub fn unexpected_try_recover(
&mut self,
t: &token::Token,
) -> PResult<'a, bool /* recovered */> {
let token_str = pprust::token_to_string(t);
let this_token_str = self.this_token_descr();
let (prev_sp, sp) = match (&self.token, self.subparser_name) {
// Point at the end of the macro call when reaching end of macro arguments.
(token::Token::Eof, Some(_)) => {
let sp = self.sess.source_map().next_point(self.span);
(sp, sp)
}
// We don't want to point at the following span after DUMMY_SP.
// This happens when the parser finds an empty TokenStream.
_ if self.prev_span == DUMMY_SP => (self.span, self.span),
// EOF, don't want to point at the following char, but rather the last token.
(token::Token::Eof, None) => (self.prev_span, self.span),
_ => (self.sess.source_map().next_point(self.prev_span), self.span),
};
let msg = format!(
"expected `{}`, found {}",
token_str,
match (&self.token, self.subparser_name) {
(token::Token::Eof, Some(origin)) => format!("end of {}", origin),
_ => this_token_str,
},
);
let mut err = self.struct_span_err(sp, &msg);
let label_exp = format!("expected `{}`", token_str);
match self.recover_closing_delimiter(&[t.clone()], err) {
Err(e) => err = e,
Ok(recovered) => {
return Ok(recovered);
}
}
let cm = self.sess.source_map();
match (cm.lookup_line(prev_sp.lo()), cm.lookup_line(sp.lo())) {
(Ok(ref a), Ok(ref b)) if a.line == b.line => {
// When the spans are in the same line, it means that the only content
// between them is whitespace, point only at the found token.
err.span_label(sp, label_exp);
}
_ => {
err.span_label(prev_sp, label_exp);
err.span_label(sp, "unexpected token");
}
}
Err(err)
}
/// Consume alternative await syntaxes like `await <expr>`, `await? <expr>`, `await(<expr>)`
/// and `await { <expr> }`.
crate fn parse_incorrect_await_syntax(
@ -562,4 +614,23 @@ impl<'a> Parser<'a> {
}
}
crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> {
let (span, msg) = match (&self.token, self.subparser_name) {
(&token::Token::Eof, Some(origin)) => {
let sp = self.sess.source_map().next_point(self.span);
(sp, format!("expected expression, found end of {}", origin))
}
_ => (self.span, format!(
"expected expression, found {}",
self.this_token_descr(),
)),
};
let mut err = self.struct_span_err(span, &msg);
let sp = self.sess.source_map().start_point(self.span);
if let Some(sp) = self.sess.ambiguous_block_expr_parse.borrow().get(&sp) {
self.sess.expr_parentheses_needed(&mut err, *sp, None);
}
err.span_label(span, "expected expression");
err
}
}

22
src/libsyntax/parse/mod.rs
View File

@ -236,7 +236,7 @@ fn maybe_source_file_to_parser(
) -> Result<Parser<'_>, Vec<Diagnostic>> {
let end_pos = source_file.end_pos;
let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?;
let mut parser = stream_to_parser(sess, stream);
let mut parser = stream_to_parser(sess, stream, None);
parser.unclosed_delims = unclosed_delims;
if parser.token == token::Eof && parser.span.is_dummy() {
parser.span = Span::new(end_pos, end_pos, parser.span.ctxt());
@ -248,7 +248,7 @@ fn maybe_source_file_to_parser(
// must preserve old name for now, because quote! from the *existing*
// compiler expands into it
pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser<'_> {
stream_to_parser(sess, tts.into_iter().collect())
stream_to_parser(sess, tts.into_iter().collect(), crate::MACRO_ARGUMENTS)
}
@ -328,8 +328,12 @@ pub fn maybe_file_to_stream(
}
/// Given stream and the `ParseSess`, produces a parser.
pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> {
Parser::new(sess, stream, None, true, false)
pub fn stream_to_parser<'a>(
sess: &'a ParseSess,
stream: TokenStream,
subparser_name: Option<&'static str>,
) -> Parser<'a> {
Parser::new(sess, stream, None, true, false, subparser_name)
}
/// Given stream, the `ParseSess` and the base directory, produces a parser.
@ -343,10 +347,12 @@ pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> {
/// The main usage of this function is outside of rustc, for those who uses
/// libsyntax as a library. Please do not remove this function while refactoring
/// just because it is not used in rustc codebase!
pub fn stream_to_parser_with_base_dir<'a>(sess: &'a ParseSess,
stream: TokenStream,
base_dir: Directory<'a>) -> Parser<'a> {
Parser::new(sess, stream, Some(base_dir), true, false)
pub fn stream_to_parser_with_base_dir<'a>(
sess: &'a ParseSess,
stream: TokenStream,
base_dir: Directory<'a>,
) -> Parser<'a> {
Parser::new(sess, stream, Some(base_dir), true, false, None)
}
/// A sequence separator.

84
src/libsyntax/parse/parser.rs
View File

@ -51,7 +51,7 @@ use crate::symbol::{kw, sym, Symbol};
use errors::{Applicability, DiagnosticBuilder, DiagnosticId, FatalError};
use rustc_target::spec::abi::{self, Abi};
use syntax_pos::{
Span, MultiSpan, BytePos, FileName,
BytePos, DUMMY_SP, FileName, MultiSpan, Span,
hygiene::CompilerDesugaringKind,
};
use log::{debug, trace};
@ -233,6 +233,8 @@ pub struct Parser<'a> {
/// error.
crate unclosed_delims: Vec<UnmatchedBrace>,
last_unexpected_token_span: Option<Span>,
/// If present, this `Parser` is not parsing Rust code but rather a macro call.
crate subparser_name: Option<&'static str>,
}
impl<'a> Drop for Parser<'a> {
@ -309,7 +311,7 @@ impl TokenCursor {
self.frame = frame;
continue
} else {
return TokenAndSpan { tok: token::Eof, sp: syntax_pos::DUMMY_SP }
return TokenAndSpan { tok: token::Eof, sp: DUMMY_SP }
};
match self.frame.last_token {
@ -533,17 +535,19 @@ enum TokenExpectType {
}
impl<'a> Parser<'a> {
pub fn new(sess: &'a ParseSess,
tokens: TokenStream,
directory: Option<Directory<'a>>,
recurse_into_file_modules: bool,
desugar_doc_comments: bool)
-> Self {
pub fn new(
sess: &'a ParseSess,
tokens: TokenStream,
directory: Option<Directory<'a>>,
recurse_into_file_modules: bool,
desugar_doc_comments: bool,
subparser_name: Option<&'static str>,
) -> Self {
let mut parser = Parser {
sess,
token: token::Whitespace,
span: syntax_pos::DUMMY_SP,
prev_span: syntax_pos::DUMMY_SP,
span: DUMMY_SP,
prev_span: DUMMY_SP,
meta_var_span: None,
prev_token_kind: PrevTokenKind::Other,
restrictions: Restrictions::empty(),
@ -568,6 +572,7 @@ impl<'a> Parser<'a> {
max_angle_bracket_count: 0,
unclosed_delims: Vec::new(),
last_unexpected_token_span: None,
subparser_name,
};
let tok = parser.next_tok();
@ -631,44 +636,13 @@ impl<'a> Parser<'a> {
}
/// Expects and consumes the token `t`. Signals an error if the next token is not `t`.
pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> {
pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> {
if self.expected_tokens.is_empty() {
if self.token == *t {
self.bump();
Ok(false)
} else {
let token_str = pprust::token_to_string(t);
let this_token_str = self.this_token_descr();
let mut err = self.fatal(&format!("expected `{}`, found {}",
token_str,
this_token_str));
let sp = if self.token == token::Token::Eof {
// EOF, don't want to point at the following char, but rather the last token
self.prev_span
} else {
self.sess.source_map().next_point(self.prev_span)
};
let label_exp = format!("expected `{}`", token_str);
match self.recover_closing_delimiter(&[t.clone()], err) {
Err(e) => err = e,
Ok(recovered) => {
return Ok(recovered);
}
}
let cm = self.sess.source_map();
match (cm.lookup_line(self.span.lo()), cm.lookup_line(sp.lo())) {
(Ok(ref a), Ok(ref b)) if a.line == b.line => {
// When the spans are in the same line, it means that the only content
// between them is whitespace, point only at the found token.
err.span_label(self.span, label_exp);
}
_ => {
err.span_label(sp, label_exp);
err.span_label(self.span, "unexpected token");
}
}
Err(err)
self.unexpected_try_recover(t)
}
} else {
self.expect_one_of(slice::from_ref(t), &[])
@ -812,7 +786,7 @@ impl<'a> Parser<'a> {
// | expected one of 8 possible tokens here
err.span_label(self.span, label_exp);
}
_ if self.prev_span == syntax_pos::DUMMY_SP => {
_ if self.prev_span == DUMMY_SP => {
// Account for macro context where the previous span might not be
// available to avoid incorrect output (#54841).
err.span_label(self.span, "unexpected token");
@ -2041,7 +2015,7 @@ impl<'a> Parser<'a> {
path = self.parse_path(PathStyle::Type)?;
path_span = path_lo.to(self.prev_span);
} else {
path = ast::Path { segments: Vec::new(), span: syntax_pos::DUMMY_SP };
path = ast::Path { segments: Vec::new(), span: DUMMY_SP };
path_span = self.span.to(self.span);
}
@ -2627,17 +2601,7 @@ impl<'a> Parser<'a> {
}
Err(mut err) => {
self.cancel(&mut err);
let msg = format!("expected expression, found {}",
self.this_token_descr());
let mut err = self.fatal(&msg);
let sp = self.sess.source_map().start_point(self.span);
if let Some(sp) = self.sess.ambiguous_block_expr_parse.borrow()
.get(&sp)
{
self.sess.expr_parentheses_needed(&mut err, *sp, None);
}
err.span_label(self.span, "expected expression");
return Err(err);
return Err(self.expected_expression_found());
}
}
}
@ -5592,7 +5556,7 @@ impl<'a> Parser<'a> {
where_clause: WhereClause {
id: ast::DUMMY_NODE_ID,
predicates: Vec::new(),
span: syntax_pos::DUMMY_SP,
span: DUMMY_SP,
},
span: span_lo.to(self.prev_span),
})
@ -5838,7 +5802,7 @@ impl<'a> Parser<'a> {
let mut where_clause = WhereClause {
id: ast::DUMMY_NODE_ID,
predicates: Vec::new(),
span: syntax_pos::DUMMY_SP,
span: DUMMY_SP,
};
if !self.eat_keyword(kw::Where) {
@ -7005,7 +6969,7 @@ impl<'a> Parser<'a> {
Ident::with_empty_ctxt(sym::warn_directory_ownership)),
tokens: TokenStream::empty(),
is_sugared_doc: false,
span: syntax_pos::DUMMY_SP,
span: DUMMY_SP,
};
attr::mark_known(&attr);
attrs.push(attr);
@ -7013,7 +6977,7 @@ impl<'a> Parser<'a> {
Ok((id, ItemKind::Mod(module), Some(attrs)))
} else {
let placeholder = ast::Mod {
inner: syntax_pos::DUMMY_SP,
inner: DUMMY_SP,
items: Vec::new(),
inline: false
};

6
src/libsyntax_ext/asm.rs
View File

@ -138,7 +138,11 @@ fn parse_inline_asm<'a>(
if p2.token != token::Eof {
let mut extra_tts = p2.parse_all_token_trees()?;
extra_tts.extend(tts[first_colon..].iter().cloned());
p = parse::stream_to_parser(cx.parse_sess, extra_tts.into_iter().collect());
p = parse::stream_to_parser(
cx.parse_sess,
extra_tts.into_iter().collect(),
Some("inline assembly"),
);
}
asm = s;

2
src/libsyntax_ext/deriving/custom.rs
View File

@ -89,7 +89,7 @@ impl MultiItemModifier for ProcMacroDerive {
let error_count_before = ecx.parse_sess.span_diagnostic.err_count();
let msg = "proc-macro derive produced unparseable tokens";
let mut parser = parse::stream_to_parser(ecx.parse_sess, stream);
let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive"));
let mut items = vec![];
loop {

12
src/test/ui/macros/format-parse-errors.stderr
View File

@ -12,17 +12,17 @@ error: expected expression, found keyword `struct`
LL | format!(struct);
| ^^^^^^ expected expression
error: expected expression, found `<eof>`
--> $DIR/format-parse-errors.rs:4:23
error: expected expression, found end of macro arguments
--> $DIR/format-parse-errors.rs:4:24
|
LL | format!("s", name =);
| ^ expected expression
| ^ expected expression
error: expected `=`, found `<eof>`
--> $DIR/format-parse-errors.rs:5:29
error: expected `=`, found end of macro arguments
--> $DIR/format-parse-errors.rs:5:32
|
LL | format!("s", foo = foo, bar);
| ^^^ expected `=`
| ^ expected `=`
error: expected expression, found keyword `struct`
--> $DIR/format-parse-errors.rs:6:24

12
src/test/ui/malformed/malformed-derive-entry.rs
View File

@ -1,17 +1,13 @@
#[derive(Copy(Bad))]
//~^ ERROR expected one of `)`, `,`, or `::`, found `(`
#[derive(Copy(Bad))] //~ ERROR expected one of `)`, `,`, or `::`, found `(`
struct Test1;
#[derive(Copy="bad")]
//~^ ERROR expected one of `)`, `,`, or `::`, found `=`
#[derive(Copy="bad")] //~ ERROR expected one of `)`, `,`, or `::`, found `=`
struct Test2;
#[derive()]
//~^ WARNING empty trait list
#[derive()] //~ WARNING empty trait list
struct Test3;
#[derive]
//~^ ERROR attribute must be of the form
#[derive] //~ ERROR attribute must be of the form `#[derive(Trait1, Trait2, ...)]`
struct Test4;
fn main() {}

6
src/test/ui/malformed/malformed-derive-entry.stderr
View File

@ -5,19 +5,19 @@ LL | #[derive(Copy(Bad))]
| ^ expected one of `)`, `,`, or `::` here
error: expected one of `)`, `,`, or `::`, found `=`
--> $DIR/malformed-derive-entry.rs:5:14
--> $DIR/malformed-derive-entry.rs:4:14
|
LL | #[derive(Copy="bad")]
| ^ expected one of `)`, `,`, or `::` here
warning: empty trait list in `derive`
--> $DIR/malformed-derive-entry.rs:9:1
--> $DIR/malformed-derive-entry.rs:7:1
|
LL | #[derive()]
| ^^^^^^^^^^^
error: attribute must be of the form `#[derive(Trait1, Trait2, ...)]`
--> $DIR/malformed-derive-entry.rs:13:1
--> $DIR/malformed-derive-entry.rs:10:1
|
LL | #[derive]
| ^^^^^^^^^

2
src/test/ui/malformed/malformed-special-attrs.rs
View File

@ -1,4 +1,4 @@
#[cfg_attr] //~ ERROR expected `(`, found `<eof>`
#[cfg_attr] //~ ERROR expected `(`, found end of attribute
struct S1;
#[cfg_attr = ""] //~ ERROR expected `(`, found `=`

11
src/test/ui/malformed/malformed-special-attrs.stderr
View File

@ -1,13 +1,14 @@
error: expected `(`, found `<eof>`
error: expected `(`, found end of attribute
--> $DIR/malformed-special-attrs.rs:1:1
|
LL | #[cfg_attr]
| ^ expected `(`
error: expected `(`, found `=`
--> $DIR/malformed-special-attrs.rs:4:12
|
LL | #[cfg_attr]
| - expected `(`
...
LL | #[cfg_attr = ""]
| ^ unexpected token
| ^ expected `(`
error: attribute must be of the form `#[derive(Trait1, Trait2, ...)]`
--> $DIR/malformed-special-attrs.rs:7:1

4
src/test/ui/parser/macro/bad-macro-argument.rs Normal file
View File

@ -0,0 +1,4 @@
fn main() {
let message = "world";
println!("Hello, {}", message/); //~ ERROR expected expression
}

8
src/test/ui/parser/macro/bad-macro-argument.stderr Normal file
View File

@ -0,0 +1,8 @@
error: expected expression, found end of macro arguments
--> $DIR/bad-macro-argument.rs:3:35
|
LL | println!("Hello, {}", message/);
| ^ expected expression
error: aborting due to previous error

2
src/test/ui/proc-macro/attr-invalid-exprs.rs
View File

@ -9,7 +9,7 @@ use attr_stmt_expr::{duplicate, no_output};
fn main() {
let _ = #[no_output] "Hello, world!";
//~^ ERROR expected expression, found `<eof>`
//~^ ERROR expected expression, found end of macro arguments
let _ = #[duplicate] "Hello, world!";
//~^ ERROR macro expansion ignores token `,` and any following

2
src/test/ui/proc-macro/attr-invalid-exprs.stderr
View File

@ -1,4 +1,4 @@
error: expected expression, found `<eof>`
error: expected expression, found end of macro arguments
--> $DIR/attr-invalid-exprs.rs:11:13
|
LL | let _ = #[no_output] "Hello, world!";