Rollup merge of #40532 - jseyfried:improve_tokenstream_quoter, r=nrc
macros: improve the `TokenStream` quoter This PR - renames the `TokenStream` quoter from `qquote!` to `quote!`, - uses `$` instead of `unquote` (e.g. `let toks: TokenStream = ...; quote!([$toks])`), - allows unquoting `Token`s as well as `TokenTree`s and `TokenStream`s (fixes #39746), and - to preserve syntactic space, requires that `$` be followed by - a single identifier to unquote, or - another `$` to produce a literal `$`. r? @nrc
This commit is contained in:
commit
7471d9793c
|
@ -13,62 +13,64 @@
|
|||
//! A library for procedural macro writers.
|
||||
//!
|
||||
//! ## Usage
|
||||
//! This crate provides the `qquote!` macro for syntax creation.
|
||||
//! This crate provides the `quote!` macro for syntax creation.
|
||||
//!
|
||||
//! The `qquote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;`
|
||||
//! The `quote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;`
|
||||
//! at the crate root. This is a temporary solution until we have better hygiene.
|
||||
//!
|
||||
//! ## Quasiquotation
|
||||
//!
|
||||
//! The quasiquoter creates output that, when run, constructs the tokenstream specified as
|
||||
//! input. For example, `qquote!(5 + 5)` will produce a program, that, when run, will
|
||||
//! input. For example, `quote!(5 + 5)` will produce a program, that, when run, will
|
||||
//! construct the TokenStream `5 | + | 5`.
|
||||
//!
|
||||
//! ### Unquoting
|
||||
//!
|
||||
//! Unquoting is currently done as `unquote`, and works by taking the single next
|
||||
//! TokenTree in the TokenStream as the unquoted term. Ergonomically, `unquote(foo)` works
|
||||
//! fine, but `unquote foo` is also supported.
|
||||
//! Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
|
||||
//! To quote `$` itself, use `$$`.
|
||||
//!
|
||||
//! A simple example might be:
|
||||
//! A simple example is:
|
||||
//!
|
||||
//!```
|
||||
//!fn double(tmp: TokenStream) -> TokenStream {
|
||||
//! qquote!(unquote(tmp) * 2)
|
||||
//! quote!($tmp * 2)
|
||||
//!}
|
||||
//!```
|
||||
//!
|
||||
//! ### Large Example: Implementing Scheme's `cond`
|
||||
//! ### Large example: Scheme's `cond`
|
||||
//!
|
||||
//! Below is the full implementation of Scheme's `cond` operator.
|
||||
//! Below is an example implementation of Scheme's `cond`.
|
||||
//!
|
||||
//! ```
|
||||
//! fn cond_rec(input: TokenStream) -> TokenStream {
|
||||
//! if input.is_empty() { return quote!(); }
|
||||
//! fn cond(input: TokenStream) -> TokenStream {
|
||||
//! let mut conds = Vec::new();
|
||||
//! let mut input = input.trees().peekable();
|
||||
//! while let Some(tree) = input.next() {
|
||||
//! let mut cond = match tree {
|
||||
//! TokenTree::Delimited(_, ref delimited) => delimited.stream(),
|
||||
//! _ => panic!("Invalid input"),
|
||||
//! };
|
||||
//! let mut trees = cond.trees();
|
||||
//! let test = trees.next();
|
||||
//! let rhs = trees.collect::<TokenStream>();
|
||||
//! if rhs.is_empty() {
|
||||
//! panic!("Invalid macro usage in cond: {}", cond);
|
||||
//! }
|
||||
//! let is_else = match test {
|
||||
//! Some(TokenTree::Token(_, Token::Ident(ident))) if ident.name == "else" => true,
|
||||
//! _ => false,
|
||||
//! };
|
||||
//! conds.push(if is_else || input.peek().is_none() {
|
||||
//! quote!({ $rhs })
|
||||
//! } else {
|
||||
//! let test = test.unwrap();
|
||||
//! quote!(if $test { $rhs } else)
|
||||
//! });
|
||||
//! }
|
||||
//!
|
||||
//! let next = input.slice(0..1);
|
||||
//! let rest = input.slice_from(1..);
|
||||
//!
|
||||
//! let clause : TokenStream = match next.maybe_delimited() {
|
||||
//! Some(ts) => ts,
|
||||
//! _ => panic!("Invalid input"),
|
||||
//! };
|
||||
//!
|
||||
//! // clause is ([test]) [rhs]
|
||||
//! if clause.len() < 2 { panic!("Invalid macro usage in cond: {:?}", clause) }
|
||||
//!
|
||||
//! let test: TokenStream = clause.slice(0..1);
|
||||
//! let rhs: TokenStream = clause.slice_from(1..);
|
||||
//!
|
||||
//! if ident_eq(&test[0], str_to_ident("else")) || rest.is_empty() {
|
||||
//! quote!({unquote(rhs)})
|
||||
//! } else {
|
||||
//! quote!({if unquote(test) { unquote(rhs) } else { cond!(unquote(rest)) } })
|
||||
//! }
|
||||
//! conds.into_iter().collect()
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
|
||||
#![crate_name = "proc_macro_plugin"]
|
||||
#![unstable(feature = "rustc_private", issue = "27812")]
|
||||
#![feature(plugin_registrar)]
|
||||
|
@ -87,8 +89,8 @@ extern crate rustc_plugin;
|
|||
extern crate syntax;
|
||||
extern crate syntax_pos;
|
||||
|
||||
mod qquote;
|
||||
use qquote::qquote;
|
||||
mod quote;
|
||||
use quote::quote;
|
||||
|
||||
use rustc_plugin::Registry;
|
||||
use syntax::ext::base::SyntaxExtension;
|
||||
|
@ -99,6 +101,6 @@ use syntax::symbol::Symbol;
|
|||
|
||||
#[plugin_registrar]
|
||||
pub fn plugin_registrar(reg: &mut Registry) {
|
||||
reg.register_syntax_extension(Symbol::intern("qquote"),
|
||||
SyntaxExtension::ProcMacro(Box::new(qquote)));
|
||||
reg.register_syntax_extension(Symbol::intern("quote"),
|
||||
SyntaxExtension::ProcMacro(Box::new(quote)));
|
||||
}
|
||||
|
|
|
@ -19,7 +19,7 @@ use syntax_pos::DUMMY_SP;
|
|||
|
||||
use std::iter;
|
||||
|
||||
pub fn qquote<'cx>(stream: TokenStream) -> TokenStream {
|
||||
pub fn quote<'cx>(stream: TokenStream) -> TokenStream {
|
||||
stream.quote()
|
||||
}
|
||||
|
||||
|
@ -72,28 +72,32 @@ impl Quote for TokenStream {
|
|||
return quote!(::syntax::tokenstream::TokenStream::empty());
|
||||
}
|
||||
|
||||
struct Quote(iter::Peekable<tokenstream::Cursor>);
|
||||
struct Quoter(iter::Peekable<tokenstream::Cursor>);
|
||||
|
||||
impl Iterator for Quote {
|
||||
impl Iterator for Quoter {
|
||||
type Item = TokenStream;
|
||||
|
||||
fn next(&mut self) -> Option<TokenStream> {
|
||||
let is_unquote = match self.0.peek() {
|
||||
Some(&TokenTree::Token(_, Token::Ident(ident))) if ident.name == "unquote" => {
|
||||
self.0.next();
|
||||
true
|
||||
let quoted_tree = if let Some(&TokenTree::Token(_, Token::Dollar)) = self.0.peek() {
|
||||
self.0.next();
|
||||
match self.0.next() {
|
||||
Some(tree @ TokenTree::Token(_, Token::Ident(..))) => Some(tree.into()),
|
||||
Some(tree @ TokenTree::Token(_, Token::Dollar)) => Some(tree.quote()),
|
||||
// FIXME(jseyfried): improve these diagnostics
|
||||
Some(..) => panic!("`$` must be followed by an ident or `$` in `quote!`"),
|
||||
None => panic!("unexpected trailing `$` in `quote!`"),
|
||||
}
|
||||
_ => false,
|
||||
} else {
|
||||
self.0.next().as_ref().map(Quote::quote)
|
||||
};
|
||||
|
||||
self.0.next().map(|tree| {
|
||||
let quoted_tree = if is_unquote { tree.into() } else { tree.quote() };
|
||||
quoted_tree.map(|quoted_tree| {
|
||||
quote!(::syntax::tokenstream::TokenStream::from((unquote quoted_tree)),)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
let quoted = Quote(self.trees().peekable()).collect::<TokenStream>();
|
||||
let quoted = Quoter(self.trees().peekable()).collect::<TokenStream>();
|
||||
quote!([(unquote quoted)].iter().cloned().collect::<::syntax::tokenstream::TokenStream>())
|
||||
}
|
||||
}
|
|
@ -162,6 +162,12 @@ impl From<TokenTree> for TokenStream {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<Token> for TokenStream {
|
||||
fn from(token: Token) -> TokenStream {
|
||||
TokenTree::Token(DUMMY_SP, token).into()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
|
||||
fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
|
||||
TokenStream::concat(iter.into_iter().map(Into::into).collect::<Vec<_>>())
|
||||
|
|
|
@ -49,9 +49,10 @@ fn cond(input: TokenStream) -> TokenStream {
|
|||
_ => false,
|
||||
};
|
||||
conds.push(if is_else || input.peek().is_none() {
|
||||
qquote!({ unquote rhs })
|
||||
quote!({ $rhs })
|
||||
} else {
|
||||
qquote!(if unquote(test.unwrap()) { unquote rhs } else)
|
||||
let test = test.unwrap();
|
||||
quote!(if $test { $rhs } else)
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
@ -29,6 +29,11 @@ pub fn plugin_registrar(reg: &mut Registry) {
|
|||
|
||||
// This macro is not very interesting, but it does contain delimited tokens with
|
||||
// no content - `()` and `{}` - which has caused problems in the past.
|
||||
// Also, it tests that we can escape `$` via `$$`.
|
||||
fn hello(_: TokenStream) -> TokenStream {
|
||||
qquote!({ fn hello() {} hello(); })
|
||||
quote!({
|
||||
fn hello() {}
|
||||
macro_rules! m { ($$($$t:tt)*) => { $$($$t)* } }
|
||||
m!(hello());
|
||||
})
|
||||
}
|
||||
|
|
|
@ -34,21 +34,21 @@ pub fn plugin_registrar(reg: &mut Registry) {
|
|||
}
|
||||
|
||||
fn attr_tru(_attr: TokenStream, _item: TokenStream) -> TokenStream {
|
||||
qquote!(fn f1() -> bool { true })
|
||||
quote!(fn f1() -> bool { true })
|
||||
}
|
||||
|
||||
fn attr_identity(_attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
qquote!(unquote item)
|
||||
quote!($item)
|
||||
}
|
||||
|
||||
fn tru(_ts: TokenStream) -> TokenStream {
|
||||
qquote!(true)
|
||||
quote!(true)
|
||||
}
|
||||
|
||||
fn ret_tru(_ts: TokenStream) -> TokenStream {
|
||||
qquote!(return true;)
|
||||
quote!(return true;)
|
||||
}
|
||||
|
||||
fn identity(ts: TokenStream) -> TokenStream {
|
||||
qquote!(unquote ts)
|
||||
quote!($ts)
|
||||
}
|
||||
|
|
|
@ -22,6 +22,6 @@ use syntax::parse::token;
|
|||
use syntax::tokenstream::TokenTree;
|
||||
|
||||
fn main() {
|
||||
let true_tok = TokenTree::Token(syntax_pos::DUMMY_SP, token::Ident(Ident::from_str("true")));
|
||||
assert!(qquote!(true).eq_unspanned(&true_tok.into()));
|
||||
let true_tok = token::Ident(Ident::from_str("true"));
|
||||
assert!(quote!(true).eq_unspanned(&true_tok.into()));
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue