Rollup merge of #40532 - jseyfried:improve_tokenstream_quoter, r=nrc

macros: improve the `TokenStream` quoter

This PR
 - renames the `TokenStream` quoter from `qquote!` to `quote!`,
 - uses `$` instead of `unquote` (e.g. `let toks: TokenStream = ...; quote!([$toks])`),
 - allows unquoting `Token`s as well as `TokenTree`s and `TokenStream`s (fixes #39746), and
 - to preserve syntactic space, requires that `$` be followed by
   - a single identifier to unquote, or
   - another `$` to produce a literal `$`.

r? @nrc

src/libproc_macro_plugin/lib.rs

@@ -13,62 +13,64 @@
 //! A library for procedural macro writers.
 //!
 //! ## Usage
-//! This crate provides the `qquote!` macro for syntax creation.
+//! This crate provides the `quote!` macro for syntax creation.
 //!
-//! The `qquote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;`
+//! The `quote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;`
 //! at the crate root. This is a temporary solution until we have better hygiene.
 //!
 //! ## Quasiquotation
 //!
 //! The quasiquoter creates output that, when run, constructs the tokenstream specified as
-//! input. For example, `qquote!(5 + 5)` will produce a program, that, when run, will
+//! input. For example, `quote!(5 + 5)` will produce a program, that, when run, will
 //! construct the TokenStream `5 | + | 5`.
 //!
 //! ### Unquoting
 //!
-//! Unquoting is currently done as `unquote`, and works by taking the single next
+//! Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
-//! TokenTree in the TokenStream as the unquoted term. Ergonomically, `unquote(foo)` works
+//! To quote `$` itself, use `$$`.
-//! fine, but `unquote foo` is also supported.
 //!
-//! A simple example might be:
+//! A simple example is:
 //!
 //!```
 //!fn double(tmp: TokenStream) -> TokenStream {
-//!    qquote!(unquote(tmp) * 2)
+//!    quote!($tmp * 2)
 //!}
 //!```
 //!
-//! ### Large Example: Implementing Scheme's `cond`
+//! ### Large example: Scheme's `cond`
 //!
-//! Below is the full implementation of Scheme's `cond` operator.
+//! Below is an example implementation of Scheme's `cond`.
 //!
 //! ```
-//! fn cond_rec(input: TokenStream) -> TokenStream {
+//! fn cond(input: TokenStream) -> TokenStream {
-//!   if input.is_empty() { return quote!(); }
+//!     let mut conds = Vec::new();
-//!
+//!     let mut input = input.trees().peekable();
-//!   let next = input.slice(0..1);
+//!     while let Some(tree) = input.next() {
-//!   let rest = input.slice_from(1..);
+//!         let mut cond = match tree {
-//!
+//!             TokenTree::Delimited(_, ref delimited) => delimited.stream(),
-//!   let clause : TokenStream = match next.maybe_delimited() {
-//!     Some(ts) => ts,
 //!             _ => panic!("Invalid input"),
 //!         };
-//!
+//!         let mut trees = cond.trees();
-//!   // clause is ([test]) [rhs]
+//!         let test = trees.next();
-//!   if clause.len() < 2 { panic!("Invalid macro usage in cond: {:?}", clause) }
+//!         let rhs = trees.collect::<TokenStream>();
-//!
+//!         if rhs.is_empty() {
-//!   let test: TokenStream = clause.slice(0..1);
+//!             panic!("Invalid macro usage in cond: {}", cond);
-//!   let rhs: TokenStream = clause.slice_from(1..);
-//!
-//!   if ident_eq(&test[0], str_to_ident("else")) || rest.is_empty() {
-//!     quote!({unquote(rhs)})
-//!   } else {
-//!     quote!({if unquote(test) { unquote(rhs) } else { cond!(unquote(rest)) } })
 //!         }
+//!         let is_else = match test {
+//!             Some(TokenTree::Token(_, Token::Ident(ident))) if ident.name == "else" => true,
+//!             _ => false,
+//!         };
+//!         conds.push(if is_else || input.peek().is_none() {
+//!             quote!({ $rhs })
+//!         } else {
+//!             let test = test.unwrap();
+//!             quote!(if $test { $rhs } else)
+//!         });
+//!     }
+//!
+//!     conds.into_iter().collect()
 //! }
 //! ```
-//!
-
 #![crate_name = "proc_macro_plugin"]
 #![unstable(feature = "rustc_private", issue = "27812")]
 #![feature(plugin_registrar)]
@@ -87,8 +89,8 @@ extern crate rustc_plugin;
 extern crate syntax;
 extern crate syntax_pos;
 
-mod qquote;
+mod quote;
-use qquote::qquote;
+use quote::quote;
 
 use rustc_plugin::Registry;
 use syntax::ext::base::SyntaxExtension;
@@ -99,6 +101,6 @@ use syntax::symbol::Symbol;
 
 #[plugin_registrar]
 pub fn plugin_registrar(reg: &mut Registry) {
-    reg.register_syntax_extension(Symbol::intern("qquote"),
+    reg.register_syntax_extension(Symbol::intern("quote"),
-                                  SyntaxExtension::ProcMacro(Box::new(qquote)));
+                                  SyntaxExtension::ProcMacro(Box::new(quote)));
 }

src/libproc_macro_plugin/quote.rs

@@ -19,7 +19,7 @@ use syntax_pos::DUMMY_SP;
 
 use std::iter;
 
-pub fn qquote<'cx>(stream: TokenStream) -> TokenStream {
+pub fn quote<'cx>(stream: TokenStream) -> TokenStream {
     stream.quote()
 }
 
@@ -72,28 +72,32 @@ impl Quote for TokenStream {
             return quote!(::syntax::tokenstream::TokenStream::empty());
         }
 
-        struct Quote(iter::Peekable<tokenstream::Cursor>);
+        struct Quoter(iter::Peekable<tokenstream::Cursor>);
 
-        impl Iterator for Quote {
+        impl Iterator for Quoter {
             type Item = TokenStream;
 
             fn next(&mut self) -> Option<TokenStream> {
-                let is_unquote = match self.0.peek() {
+                let quoted_tree = if let Some(&TokenTree::Token(_, Token::Dollar)) = self.0.peek() {
-                    Some(&TokenTree::Token(_, Token::Ident(ident))) if ident.name == "unquote" => {
                     self.0.next();
-                        true
+                    match self.0.next() {
+                        Some(tree @ TokenTree::Token(_, Token::Ident(..))) => Some(tree.into()),
+                        Some(tree @ TokenTree::Token(_, Token::Dollar)) => Some(tree.quote()),
+                        // FIXME(jseyfried): improve these diagnostics
+                        Some(..) => panic!("`$` must be followed by an ident or `$` in `quote!`"),
+                        None => panic!("unexpected trailing `$` in `quote!`"),
                     }
-                    _ => false,
+                } else {
+                    self.0.next().as_ref().map(Quote::quote)
                 };
 
-                self.0.next().map(|tree| {
+                quoted_tree.map(|quoted_tree| {
-                    let quoted_tree = if is_unquote { tree.into() } else { tree.quote() };
                     quote!(::syntax::tokenstream::TokenStream::from((unquote quoted_tree)),)
                 })
             }
         }
 
-        let quoted = Quote(self.trees().peekable()).collect::<TokenStream>();
+        let quoted = Quoter(self.trees().peekable()).collect::<TokenStream>();
         quote!([(unquote quoted)].iter().cloned().collect::<::syntax::tokenstream::TokenStream>())
     }
 }

src/libsyntax/tokenstream.rs

@@ -162,6 +162,12 @@ impl From<TokenTree> for TokenStream {
     }
 }
 
+impl From<Token> for TokenStream {
+    fn from(token: Token) -> TokenStream {
+        TokenTree::Token(DUMMY_SP, token).into()
+    }
+}
+
 impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
     fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
         TokenStream::concat(iter.into_iter().map(Into::into).collect::<Vec<_>>())

src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs

@@ -49,9 +49,10 @@ fn cond(input: TokenStream) -> TokenStream {
             _ => false,
         };
         conds.push(if is_else || input.peek().is_none() {
-            qquote!({ unquote rhs })
+            quote!({ $rhs })
         } else {
-            qquote!(if unquote(test.unwrap()) { unquote rhs } else)
+            let test = test.unwrap();
+            quote!(if $test { $rhs } else)
         });
     }
 

src/test/run-pass-fulldeps/auxiliary/hello_macro.rs

@@ -29,6 +29,11 @@ pub fn plugin_registrar(reg: &mut Registry) {
 
 // This macro is not very interesting, but it does contain delimited tokens with
 // no content - `()` and `{}` - which has caused problems in the past.
+// Also, it tests that we can escape `$` via `$$`.
 fn hello(_: TokenStream) -> TokenStream {
-    qquote!({ fn hello() {} hello(); })
+    quote!({
+        fn hello() {}
+        macro_rules! m { ($$($$t:tt)*) => { $$($$t)* } }
+        m!(hello());
+    })
 }

src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs

@@ -34,21 +34,21 @@ pub fn plugin_registrar(reg: &mut Registry) {
 }
 
 fn attr_tru(_attr: TokenStream, _item: TokenStream) -> TokenStream {
-    qquote!(fn f1() -> bool { true })
+    quote!(fn f1() -> bool { true })
 }
 
 fn attr_identity(_attr: TokenStream, item: TokenStream) -> TokenStream {
-    qquote!(unquote item)
+    quote!($item)
 }
 
 fn tru(_ts: TokenStream) -> TokenStream {
-    qquote!(true)
+    quote!(true)
 }
 
 fn ret_tru(_ts: TokenStream) -> TokenStream {
-    qquote!(return true;)
+    quote!(return true;)
 }
 
 fn identity(ts: TokenStream) -> TokenStream {
-    qquote!(unquote ts)
+    quote!($ts)
 }

src/test/run-pass-fulldeps/macro-quote-1.rs

@@ -22,6 +22,6 @@ use syntax::parse::token;
 use syntax::tokenstream::TokenTree;
 
 fn main() {
-    let true_tok = TokenTree::Token(syntax_pos::DUMMY_SP, token::Ident(Ident::from_str("true")));
+    let true_tok = token::Ident(Ident::from_str("true"));
-    assert!(qquote!(true).eq_unspanned(&true_tok.into()));
+    assert!(quote!(true).eq_unspanned(&true_tok.into()));
 }