From 1823dea7df9e125022dfda8126b6713f142f0e73 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Fri, 21 Aug 2020 18:18:04 -0400 Subject: [PATCH] Attach `TokenStream` to `ast::Ty` A `Ty` does not have outer attributes, so we only capture tokens when parsing a `macro_rules!` matcher --- compiler/rustc_ast/src/ast.rs | 8 +++++--- compiler/rustc_ast/src/mut_visit.rs | 2 +- compiler/rustc_ast_lowering/src/lib.rs | 1 + compiler/rustc_builtin_macros/src/concat_idents.rs | 1 + compiler/rustc_expand/src/base.rs | 1 + compiler/rustc_expand/src/build.rs | 2 +- compiler/rustc_expand/src/placeholders.rs | 12 ++++++++---- compiler/rustc_parse/src/lib.rs | 1 + compiler/rustc_parse/src/parser/diagnostics.rs | 9 +++++++-- compiler/rustc_parse/src/parser/item.rs | 9 +++++++-- compiler/rustc_parse/src/parser/nonterminal.rs | 9 ++++++++- compiler/rustc_parse/src/parser/ty.rs | 2 +- 12 files changed, 42 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 8fbf5ccf5ac..e55a5b6b9a2 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -587,7 +587,7 @@ impl Pat { _ => return None, }; - Some(P(Ty { kind, id: self.id, span: self.span })) + Some(P(Ty { kind, id: self.id, span: self.span, tokens: None })) } /// Walk top-down and call `it` in each place where a pattern occurs @@ -1169,7 +1169,7 @@ impl Expr { _ => return None, }; - Some(P(Ty { kind, id: self.id, span: self.span })) + Some(P(Ty { kind, id: self.id, span: self.span, tokens: None })) } pub fn precedence(&self) -> ExprPrecedence { @@ -1867,6 +1867,7 @@ pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -2145,7 +2146,7 @@ impl Param { /// Builds a `Param` object from `ExplicitSelf`. pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param { let span = eself.span.to(eself_ident.span); - let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span }); + let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None }); let param = |mutbl, ty| Param { attrs, pat: P(Pat { @@ -2168,6 +2169,7 @@ impl Param { id: DUMMY_NODE_ID, kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }), span, + tokens: None, }), ), } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 3e86528fcef..ce2bc75fdab 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -451,7 +451,7 @@ pub fn noop_visit_ty_constraint( } pub fn noop_visit_ty(ty: &mut P, vis: &mut T) { - let Ty { id, kind, span } = ty.deref_mut(); + let Ty { id, kind, span, tokens: _ } = ty.deref_mut(); vis.visit_id(id); match kind { TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {} diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 586355fe613..20f3d551571 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -1106,6 +1106,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { id: node_id, kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()), span: constraint.span, + tokens: None, }, itctx, ); diff --git a/compiler/rustc_builtin_macros/src/concat_idents.rs b/compiler/rustc_builtin_macros/src/concat_idents.rs index 8223cdda072..c4d1c6eee31 100644 --- a/compiler/rustc_builtin_macros/src/concat_idents.rs +++ b/compiler/rustc_builtin_macros/src/concat_idents.rs @@ -61,6 +61,7 @@ pub fn expand_concat_idents<'cx>( id: ast::DUMMY_NODE_ID, kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)), span: self.ident.span, + tokens: None, })) } } diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 4c01cb8159a..5bdc4760dcc 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -607,6 +607,7 @@ impl DummyResult { id: ast::DUMMY_NODE_ID, kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, span: sp, + tokens: None, }) } } diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index 717c4e9406c..c57719d6716 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -54,7 +54,7 @@ impl<'a> ExtCtxt<'a> { } pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { - P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind }) + P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind, tokens: None }) } pub fn ty_path(&self, path: ast::Path) -> P { diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs index dbd2e70af6a..f642558d5e0 100644 --- a/compiler/rustc_expand/src/placeholders.rs +++ b/compiler/rustc_expand/src/placeholders.rs @@ -37,7 +37,8 @@ pub fn placeholder( tokens: None, }) }; - let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span }); + let ty = + || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span, tokens: None }); let pat = || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span, tokens: None }); @@ -88,9 +89,12 @@ pub fn placeholder( kind: ast::PatKind::MacCall(mac_placeholder()), tokens: None, })), - AstFragmentKind::Ty => { - AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::MacCall(mac_placeholder()) })) - } + AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { + id, + span, + kind: ast::TyKind::MacCall(mac_placeholder()), + tokens: None, + })), AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ let mac = P(ast::MacCallStmt { mac: mac_placeholder(), diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 65ded67dcf6..19b9c998320 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -270,6 +270,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke } Nonterminal::NtBlock(ref block) => block.tokens.clone(), Nonterminal::NtPat(ref pat) => pat.tokens.clone(), + Nonterminal::NtTy(ref ty) => ty.tokens.clone(), Nonterminal::NtIdent(ident, is_raw) => { Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) } diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 12efe391fb9..364c859147a 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -28,7 +28,7 @@ pub(super) fn dummy_arg(ident: Ident) -> Param { span: ident.span, tokens: None, }); - let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID }; + let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID, tokens: None }; Param { attrs: AttrVec::default(), id: ast::DUMMY_NODE_ID, @@ -75,7 +75,12 @@ impl RecoverQPath for Ty { Some(P(self.clone())) } fn recovered(qself: Option, path: ast::Path) -> Self { - Self { span: path.span, kind: TyKind::Path(qself, path), id: ast::DUMMY_NODE_ID } + Self { + span: path.span, + kind: TyKind::Path(qself, path), + id: ast::DUMMY_NODE_ID, + tokens: None, + } } } diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 1a428f8bb0a..1c4bb4532eb 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -510,7 +510,12 @@ impl<'a> Parser<'a> { { let span = self.prev_token.span.between(self.token.span); self.struct_span_err(span, "missing trait in a trait impl").emit(); - P(Ty { kind: TyKind::Path(None, err_path(span)), span, id: DUMMY_NODE_ID }) + P(Ty { + kind: TyKind::Path(None, err_path(span)), + span, + id: DUMMY_NODE_ID, + tokens: None, + }) } else { self.parse_ty()? }; @@ -1046,7 +1051,7 @@ impl<'a> Parser<'a> { // The user intended that the type be inferred, // so treat this as if the user wrote e.g. `const A: _ = expr;`. - P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID }) + P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID, tokens: None }) } /// Parses an enum declaration. diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 47ed98b8599..d70fa532850 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -141,7 +141,14 @@ impl<'a> Parser<'a> { token::NtExpr(expr) } NonterminalKind::Literal => token::NtLiteral(self.parse_literal_maybe_minus()?), - NonterminalKind::Ty => token::NtTy(self.parse_ty()?), + NonterminalKind::Ty => { + let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?; + // We have an eaten an NtTy, which could already have tokens + if ty.tokens.is_none() { + ty.tokens = Some(tokens); + } + token::NtTy(ty) + } // this could be handled like a token, since it is one NonterminalKind::Ident => { if let Some((ident, is_raw)) = get_macro_ident(&self.token) { diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs index 4356850818e..259764a317d 100644 --- a/compiler/rustc_parse/src/parser/ty.rs +++ b/compiler/rustc_parse/src/parser/ty.rs @@ -626,6 +626,6 @@ impl<'a> Parser<'a> { } pub(super) fn mk_ty(&self, span: Span, kind: TyKind) -> P { - P(Ty { kind, span, id: ast::DUMMY_NODE_ID }) + P(Ty { kind, span, id: ast::DUMMY_NODE_ID, tokens: None }) } }