Clean up ext::tt::transcribe.

This commit is contained in:
Jeffrey Seyfried 2017-01-14 12:15:26 +00:00
parent debcbf0b8e
commit 6a9248fc15
2 changed files with 35 additions and 74 deletions

View File

@ -16,7 +16,7 @@ use ext::expand::{Expansion, ExpansionKind};
use ext::tt::macro_parser::{Success, Error, Failure};
use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
use ext::tt::macro_parser::{parse, parse_failure_msg};
use ext::tt::transcribe::new_tt_reader;
use ext::tt::transcribe::transcribe;
use parse::{Directory, ParseSess};
use parse::parser::Parser;
use parse::token::{self, NtTT, Token};
@ -113,16 +113,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
_ => cx.span_bug(sp, "malformed macro rhs"),
};
// rhs has holes ( `$id` and `$(...)` that need filled)
let mut trncbr =
new_tt_reader(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs);
let mut tts = Vec::new();
loop {
let tok = trncbr.real_token();
if tok.tok == token::Eof {
break
}
tts.push(TokenTree::Token(tok.sp, tok.tok));
}
let tts = transcribe(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs);
let directory = Directory {
path: cx.current_expansion.module.directory.clone(),
ownership: cx.current_expansion.directory_ownership,

View File

@ -13,7 +13,6 @@ use ast::Ident;
use errors::Handler;
use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
use parse::token::{self, MatchNt, SubstNt, Token, NtIdent};
use parse::lexer::TokenAndSpan;
use syntax_pos::{Span, DUMMY_SP};
use tokenstream::{self, TokenTree};
use util::small_vector::SmallVector;
@ -32,8 +31,8 @@ struct TtFrame {
}
#[derive(Clone)]
pub struct TtReader<'a> {
pub sp_diag: &'a Handler,
struct TtReader<'a> {
sp_diag: &'a Handler,
/// the unzipped tree:
stack: SmallVector<TtFrame>,
/* for MBE-style macro transcription */
@ -41,24 +40,15 @@ pub struct TtReader<'a> {
repeat_idx: Vec<usize>,
repeat_len: Vec<usize>,
/* cached: */
pub cur_tok: Token,
pub cur_span: Span,
}
impl<'a> TtReader<'a> {
pub fn real_token(&mut self) -> TokenAndSpan {
tt_next_token(self)
}
}
/// This can do Macro-By-Example transcription. On the other hand, if
/// `src` contains no `TokenTree::Sequence`s, `MatchNt`s or `SubstNt`s, `interp` can
/// (and should) be None.
pub fn new_tt_reader(sp_diag: &Handler,
interp: Option<HashMap<Ident, Rc<NamedMatch>>>,
src: Vec<tokenstream::TokenTree>)
-> TtReader {
pub fn transcribe(sp_diag: &Handler,
interp: Option<HashMap<Ident, Rc<NamedMatch>>>,
src: Vec<tokenstream::TokenTree>)
-> Vec<TokenTree> {
let mut r = TtReader {
sp_diag: sp_diag,
stack: SmallVector::one(TtFrame {
@ -77,12 +67,15 @@ pub fn new_tt_reader(sp_diag: &Handler,
},
repeat_idx: Vec::new(),
repeat_len: Vec::new(),
/* dummy values, never read: */
cur_tok: token::Eof,
cur_span: DUMMY_SP,
};
tt_next_token(&mut r); /* get cur_tok and cur_span set up */
r
let mut tts = Vec::new();
let mut prev_span = DUMMY_SP;
while let Some(tt) = tt_next_token(&mut r, prev_span) {
prev_span = tt.span();
tts.push(tt);
}
tts
}
fn lookup_cur_matched_by_matched(r: &TtReader, start: Rc<NamedMatch>) -> Rc<NamedMatch> {
@ -156,38 +149,24 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
/// Return the next token from the TtReader.
/// EFFECT: advances the reader's token field
pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
// FIXME(pcwalton): Bad copy?
let ret_val = TokenAndSpan {
tok: r.cur_tok.clone(),
sp: r.cur_span.clone(),
};
fn tt_next_token(r: &mut TtReader, prev_span: Span) -> Option<TokenTree> {
loop {
let should_pop = match r.stack.last() {
None => {
assert_eq!(ret_val.tok, token::Eof);
return ret_val;
}
Some(frame) => {
if frame.idx < frame.forest.len() {
break;
}
!frame.dotdotdoted ||
*r.repeat_idx.last().unwrap() == *r.repeat_len.last().unwrap() - 1
let should_pop = if let Some(frame) = r.stack.last() {
if frame.idx < frame.forest.len() {
break;
}
!frame.dotdotdoted || *r.repeat_idx.last().unwrap() == *r.repeat_len.last().unwrap() - 1
} else {
return None;
};
/* done with this set; pop or repeat? */
if should_pop {
let prev = r.stack.pop().unwrap();
match r.stack.last_mut() {
None => {
r.cur_tok = token::Eof;
return ret_val;
}
Some(frame) => {
frame.idx += 1;
}
if let Some(frame) = r.stack.last_mut() {
frame.idx += 1;
} else {
return None;
}
if prev.dotdotdoted {
r.repeat_idx.pop();
@ -197,8 +176,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
*r.repeat_idx.last_mut().unwrap() += 1;
r.stack.last_mut().unwrap().idx = 0;
if let Some(tk) = r.stack.last().unwrap().sep.clone() {
r.cur_tok = tk; // repeat same span, I guess
return ret_val;
return Some(TokenTree::Token(prev_span, tk)); // repeat same span, I guess
}
}
}
@ -234,7 +212,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
}
r.stack.last_mut().unwrap().idx += 1;
return tt_next_token(r);
return tt_next_token(r, prev_span);
}
r.repeat_len.push(len);
r.repeat_idx.push(0);
@ -252,9 +230,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
r.stack.last_mut().unwrap().idx += 1;
match lookup_cur_matched(r, ident) {
None => {
r.cur_span = sp;
r.cur_tok = SubstNt(ident);
return ret_val;
return Some(TokenTree::Token(sp, SubstNt(ident)));
// this can't be 0 length, just like TokenTree::Delimited
}
Some(cur_matched) => if let MatchedNonterminal(ref nt) = *cur_matched {
@ -263,15 +239,11 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
// (a) idents can be in lots of places, so it'd be a pain
// (b) we actually can, since it's a token.
NtIdent(ref sn) => {
r.cur_span = sn.span;
r.cur_tok = token::Ident(sn.node);
return ret_val;
return Some(TokenTree::Token(sn.span, token::Ident(sn.node)));
}
_ => {
// FIXME(pcwalton): Bad copy.
r.cur_span = sp;
r.cur_tok = token::Interpolated(nt.clone());
return ret_val;
// FIXME(pcwalton): Bad copy
return Some(TokenTree::Token(sp, token::Interpolated(nt.clone())));
}
}
} else {
@ -292,11 +264,9 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
});
// if this could be 0-length, we'd need to potentially recur here
}
TokenTree::Token(sp, tok) => {
r.cur_span = sp;
r.cur_tok = tok;
tt @ TokenTree::Token(..) => {
r.stack.last_mut().unwrap().idx += 1;
return ret_val;
return Some(tt);
}
}
}