From 3815e91ccd8c7c8ed564476425bca6f1dc5e9607 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Fri, 21 Aug 2020 18:37:34 -0400 Subject: [PATCH] Attach tokens to `NtMeta` (`ast::AttrItem`) An `AttrItem` does not have outer attributes, so we only capture tokens when parsing a `macro_rules!` matcher --- compiler/rustc_ast/src/ast.rs | 1 + compiler/rustc_ast/src/attr/mod.rs | 2 +- compiler/rustc_ast/src/mut_visit.rs | 4 ++-- compiler/rustc_ast_lowering/src/lib.rs | 1 + compiler/rustc_builtin_macros/src/cmdline_attrs.rs | 2 +- compiler/rustc_expand/src/expand.rs | 1 + compiler/rustc_parse/src/lib.rs | 1 + compiler/rustc_parse/src/parser/attr.rs | 2 +- compiler/rustc_parse/src/parser/nonterminal.rs | 9 ++++++++- 9 files changed, 17 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index e55a5b6b9a2..6db84f0dbd3 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -2419,6 +2419,7 @@ impl rustc_serialize::Decodable for AttrId { pub struct AttrItem { pub path: Path, pub args: MacArgs, + pub tokens: Option, } /// A list of attributes. diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index d819e3cb815..33bd9335231 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -330,7 +330,7 @@ crate fn mk_attr_id() -> AttrId { } pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute { - mk_attr_from_item(style, AttrItem { path, args }, span) + mk_attr_from_item(style, AttrItem { path, args, tokens: None }, span) } pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute { diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index ce2bc75fdab..3ba8ff65ee5 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -579,7 +579,7 @@ pub fn noop_visit_local(local: &mut P, vis: &mut T) { pub fn noop_visit_attribute(attr: &mut Attribute, vis: &mut T) { let Attribute { kind, id: _, style: _, span } = attr; match kind { - AttrKind::Normal(AttrItem { path, args }) => { + AttrKind::Normal(AttrItem { path, args, tokens: _ }) => { vis.visit_path(path); visit_mac_args(args, vis); } @@ -709,7 +709,7 @@ pub fn noop_visit_interpolated(nt: &mut token::Nonterminal, vis: token::NtLifetime(ident) => vis.visit_ident(ident), token::NtLiteral(expr) => vis.visit_expr(expr), token::NtMeta(item) => { - let AttrItem { path, args } = item.deref_mut(); + let AttrItem { path, args, tokens: _ } = item.deref_mut(); vis.visit_path(path); visit_mac_args(args, vis); } diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 20f3d551571..a28d022c661 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -967,6 +967,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { AttrKind::Normal(ref item) => AttrKind::Normal(AttrItem { path: item.path.clone(), args: self.lower_mac_args(&item.args), + tokens: None, }), AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data), }; diff --git a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs index 34e2accc615..5ed8b69d92a 100644 --- a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs +++ b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs @@ -15,7 +15,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - ); let start_span = parser.token.span; - let AttrItem { path, args } = match parser.parse_attr_item() { + let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item() { Ok(ai) => ai, Err(mut err) => { err.emit(); diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 241566a042a..98faa8a0a0f 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -1777,6 +1777,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { kind: ast::AttrKind::Normal(AttrItem { path: meta.path, args: meta.kind.mac_args(meta.span), + tokens: None, }), span: at.span, id: at.id, diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index df9b87fe7d1..bf954783d82 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -277,6 +277,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke Nonterminal::NtLifetime(ident) => { Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) } + Nonterminal::NtMeta(ref attr) => attr.tokens.clone(), Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => { if expr.tokens.is_none() { diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 4e4429e461f..98f94098bfc 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -162,7 +162,7 @@ impl<'a> Parser<'a> { } else { let path = self.parse_path(PathStyle::Mod)?; let args = self.parse_attr_args()?; - ast::AttrItem { path, args } + ast::AttrItem { path, args, tokens: None } }) } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 516c2c77213..8717f4b9245 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -169,7 +169,14 @@ impl<'a> Parser<'a> { } } NonterminalKind::Path => token::NtPath(self.parse_path(PathStyle::Type)?), - NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item()?)), + NonterminalKind::Meta => { + let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?; + // We may have eaten a nonterminal, which could already have tokens + if attr.tokens.is_none() { + attr.tokens = Some(tokens); + } + token::NtMeta(P(attr)) + } NonterminalKind::TT => token::NtTT(self.parse_token_tree()), NonterminalKind::Vis => token::NtVis(self.parse_visibility(FollowedByType::Yes)?), NonterminalKind::Lifetime => {