From 9b2f7624ecb743e9db8e135113f396a7956623e7 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 10 Jul 2017 17:44:46 -0700 Subject: [PATCH] syntax: Add `tokens: Option` to Item This commit adds a new field to the `Item` AST node in libsyntax to optionally contain the original token stream that the item itself was parsed from. This is currently `None` everywhere but is intended for use later with procedural macros. --- src/librustc_metadata/cstore_impl.rs | 1 + src/libsyntax/ast.rs | 6 ++++++ src/libsyntax/diagnostics/plugin.rs | 1 + src/libsyntax/ext/build.rs | 6 ++++-- src/libsyntax/ext/expand.rs | 1 + src/libsyntax/ext/placeholders.rs | 1 + src/libsyntax/fold.rs | 8 ++++++-- src/libsyntax/parse/parser.rs | 1 + src/libsyntax/std_inject.rs | 2 ++ src/libsyntax/test.rs | 16 +++++++++++----- src/libsyntax_ext/global_asm.rs | 1 + 11 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs index 5b0612ddab6..25079613e58 100644 --- a/src/librustc_metadata/cstore_impl.rs +++ b/src/librustc_metadata/cstore_impl.rs @@ -389,6 +389,7 @@ impl CrateStore for cstore::CStore { legacy: def.legacy, }), vis: ast::Visibility::Inherited, + tokens: None, }) } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 4fc73787353..bd26ab5bd35 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -1812,6 +1812,12 @@ pub struct Item { pub node: ItemKind, pub vis: Visibility, pub span: Span, + + /// Original tokens this item was parsed from. This isn't necessarily + /// available for all items, although over time more and more items should + /// have this be `Some`. Right now this is primarily used for procedural + /// macros, notably custom attributes. + pub tokens: Option, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 2a5de3c7382..855f4cd3557 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -236,6 +236,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt, ), vis: ast::Visibility::Public, span: span, + tokens: None, }) ])) } diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index e004f7354eb..de0538e38b3 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -979,7 +979,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> { id: ast::DUMMY_NODE_ID, node: node, vis: ast::Visibility::Inherited, - span: span + span: span, + tokens: None, }) } @@ -1147,7 +1148,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> { attrs: vec![], node: ast::ItemKind::Use(vp), vis: vis, - span: sp + span: sp, + tokens: None, }) } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index f6d56557166..16c264e0f94 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -214,6 +214,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { ident: keywords::Invalid.ident(), id: ast::DUMMY_NODE_ID, vis: ast::Visibility::Public, + tokens: None, }))); match self.expand(krate_item).make_items().pop().map(P::unwrap) { diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs index 4fb138d506a..9bea641b033 100644 --- a/src/libsyntax/ext/placeholders.rs +++ b/src/libsyntax/ext/placeholders.rs @@ -46,6 +46,7 @@ pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion { ExpansionKind::Items => Expansion::Items(SmallVector::one(P(ast::Item { id: id, span: span, ident: ident, vis: vis, attrs: attrs, node: ast::ItemKind::Mac(mac_placeholder()), + tokens: None, }))), ExpansionKind::TraitItems => Expansion::TraitItems(SmallVector::one(ast::TraitItem { id: id, span: span, ident: ident, attrs: attrs, diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 8c616df858a..71802d0aa28 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -1000,6 +1000,7 @@ pub fn noop_fold_crate(Crate {module, attrs, span}: Crate, vis: ast::Visibility::Public, span: span, node: ast::ItemKind::Mod(module), + tokens: None, })).into_iter(); let (module, attrs, span) = match items.next() { @@ -1032,7 +1033,7 @@ pub fn noop_fold_item(i: P, folder: &mut T) -> SmallVector(Item {id, ident, attrs, node, vis, span}: Item, +pub fn noop_fold_item_simple(Item {id, ident, attrs, node, vis, span, tokens}: Item, folder: &mut T) -> Item { Item { id: folder.new_id(id), @@ -1040,7 +1041,10 @@ pub fn noop_fold_item_simple(Item {id, ident, attrs, node, vis, span} ident: folder.fold_ident(ident), attrs: fold_attrs(attrs, folder), node: folder.fold_item_kind(node), - span: folder.new_span(span) + span: folder.new_span(span), + tokens: tokens.map(|tokens| { + folder.fold_tts(tokens.into()).into() + }), } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index d1591a219b3..4f8d85a8da4 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -4653,6 +4653,7 @@ impl<'a> Parser<'a> { node: node, vis: vis, span: span, + tokens: None, // TODO: fill this in }) } diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs index a8a9ae556f1..d9ed96f293a 100644 --- a/src/libsyntax/std_inject.rs +++ b/src/libsyntax/std_inject.rs @@ -60,6 +60,7 @@ pub fn maybe_inject_crates_ref(mut krate: ast::Crate, alt_std_name: Option - folded.map(|ast::Item {id, ident, attrs, node, vis, span}| { + folded.map(|ast::Item {id, ident, attrs, node, vis, span, tokens}| { let allow_str = Symbol::intern("allow"); let dead_code_str = Symbol::intern("dead_code"); let word_vec = vec![attr::mk_list_word_item(dead_code_str)]; @@ -212,7 +212,8 @@ impl fold::Folder for EntryPointCleaner { .collect(), node: node, vis: vis, - span: span + span: span, + tokens: tokens, } }), EntryPointType::None | @@ -255,6 +256,7 @@ fn mk_reexport_mod(cx: &mut TestCtxt, node: ast::ItemKind::Mod(reexport_mod), vis: ast::Visibility::Public, span: DUMMY_SP, + tokens: None, })).pop().unwrap(); (it, sym) @@ -465,7 +467,8 @@ fn mk_std(cx: &TestCtxt) -> P { node: vi, attrs: vec![], vis: vis, - span: sp + span: sp, + tokens: None, }) } @@ -506,7 +509,8 @@ fn mk_main(cx: &mut TestCtxt) -> P { id: ast::DUMMY_NODE_ID, node: main, vis: ast::Visibility::Public, - span: sp + span: sp, + tokens: None, }) } @@ -536,6 +540,7 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P, Option>) { node: item_, vis: ast::Visibility::Public, span: DUMMY_SP, + tokens: None, })).pop().unwrap(); let reexport = cx.reexport_test_harness_main.map(|s| { // building `use = __test::main` @@ -551,7 +556,8 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P, Option>) { attrs: vec![], node: ast::ItemKind::Use(P(use_path)), vis: ast::Visibility::Inherited, - span: DUMMY_SP + span: DUMMY_SP, + tokens: None, })).pop().unwrap() }); diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs index dc67e1c45f6..8b0bb8cb891 100644 --- a/src/libsyntax_ext/global_asm.rs +++ b/src/libsyntax_ext/global_asm.rs @@ -61,5 +61,6 @@ pub fn expand_global_asm<'cx>(cx: &'cx mut ExtCtxt, })), vis: ast::Visibility::Inherited, span: sp, + tokens: None, }))) }