OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Start letting the parser catch interpolated ASTs.

This commit is contained in:
Paul Stansifer 2012-07-03 18:39:37 -07:00
parent f940653720
commit 62db5706e6
6 changed files with 95 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/libsyntax/ext/tt/earley_parser.rs
View File

@ -259,7 +259,11 @@ fn parse_nt(p: parser, name: str) -> whole_nt {
"expr" { token::w_expr(p.parse_expr()) }
"ty" { token::w_ty(p.parse_ty(false /* no need to disambiguate*/)) }
// this could be handled like a token, since it is one
"ident" { token::w_ident(p.parse_ident()) }
"ident" { alt copy p.token {
token::IDENT(sn,b) { p.bump(); token::w_ident(sn,b) }
_ { p.fatal("expected ident, found "
+ token::to_str(*p.reader.interner(), copy p.token)) }
} }
"path" { token::w_path(p.parse_path_with_tps(false)) }
_ { p.fatal("Unsupported builtin nonterminal parser: " + name)}
}

11
src/libsyntax/ext/tt/transcribe.rs
View File

@ -3,7 +3,7 @@ import diagnostic::span_handler;
import ast::{token_tree,tt_delim,tt_flat,tt_dotdotdot,tt_interpolate,ident};
import earley_parser::{arb_depth,seq,leaf};
import codemap::span;
import parse::token::{EOF,ACTUALLY,token};
import parse::token::{EOF,ACTUALLY,IDENT,token,w_ident};
import std::map::{hashmap,box_str_hash};
export tt_reader, new_tt_reader, dup_tt_reader, tt_next_token;
@ -193,8 +193,17 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
// TODO: think about span stuff here
tt_interpolate(sp, ident) {
alt *lookup_cur_ad(r, ident) {
/* sidestep the interpolation tricks for ident because
(a) idents can be in lots of places, so it'd be a pain
(b) we actually can, since it's a token. */
leaf(w_ident(sn,b)) {
r.cur_span = sp; r.cur_tok = IDENT(sn,b);
r.cur.idx += 1u;
ret ret_val;
}
leaf(w_nt) {
r.cur_span = sp; r.cur_tok = ACTUALLY(w_nt);
r.cur.idx += 1u;
ret ret_val;
}
seq(*) {

2
src/libsyntax/parse/common.rs
View File

@ -50,6 +50,8 @@ impl parser_common for parser {
fn parse_ident() -> ast::ident {
alt copy self.token {
token::IDENT(i, _) { self.bump(); ret self.get_str(i); }
token::ACTUALLY(token::w_ident(*)) { self.bug(
"ident interpolation not converted to real token"); }
_ { self.fatal("expecting ident, found "
+ token_to_str(self.reader, self.token)); }
}

3
src/libsyntax/parse/lexer.rs
View File

@ -459,8 +459,7 @@ fn next_token_inner(rdr: string_reader) -> token::token {
let is_mod_name = c == ':' && nextch(rdr) == ':';
// FIXME: perform NFKC normalization here. (Issue #2253)
ret token::IDENT(intern(*rdr.interner,
@accum_str), is_mod_name);
ret token::IDENT(intern(*rdr.interner, @accum_str), is_mod_name);
}
if is_dec_digit(c) {
ret scan_number(c, rdr);

77
src/libsyntax/parse/parser.rs
View File

@ -1,7 +1,7 @@
import result::result;
import either::{either, left, right};
import std::map::{hashmap, str_hash};
import token::{can_begin_expr, is_ident, is_plain_ident};
import token::{can_begin_expr, is_ident, is_plain_ident, ACTUALLY};
import codemap::{span,fss_none};
import util::interner;
import ast_util::{spanned, respan, mk_sp, ident_to_path, operator_prec};
@ -59,6 +59,78 @@ enum class_contents { ctor_decl(fn_decl, blk, codemap::span),
type arg_or_capture_item = either<arg, capture_item>;
type item_info = (ident, item_, option<~[attribute]>);
fn dummy() {
/* We need to position the macros to capture the ACTUALLY tokens before they
get bumped away. So two bumps in a row is bad. (the first lookahead also
counts as a bump).
Events happen L to R; 'B' indicates a bump before hand:
._____________________________.________________________.
↓ B| |
parse_expr -> parse_expr_res -> parse_assign_expr |
↓ |
parse_binops -> parse_more_binops |
↓ B↓ B|
parse_prefix_expr B-> parse_dot_or_call_expr
B|_↑ ↓
parse_bottom_expr
B↓
⋯->parse_ident
...so we've hit parse_prefix_expr, parse_more_binops, and parse_bottom_expr.
*/
#macro[[#maybe_whole_item[p],
alt copy p.token {
ACTUALLY(token::w_item(i)) { p.bump(); ret i; }
_ {} }]];
#macro[[#maybe_whole_block[p],
alt copy p.token {
ACTUALLY(token::w_block(b)) { p.bump(); ret b; }
_ {} }]];
#macro[[#maybe_whole_stmt[p],
alt copy p.token {
ACTUALLY(token::w_stmt(s)) { p.bump(); ret s; }
_ {} }]];
#macro[[#maybe_whole_pat[p],
alt copy p.token {
ACTUALLY(token::w_pat(pt)) { p.bump(); ret pt; }
_ {} }]];
#macro[[#maybe_whole_expr[p],
alt copy p.token {
ACTUALLY(token::w_expr(e)) {
p.bump();
ret e;
}
ACTUALLY(token::w_path(pt)) {
p.bump();
ret p.mk_expr(p.span.lo, p.span.lo,
expr_path(pt));
}
_ {} }]];
#macro[[#maybe_whole_expr_pexpr[p], /* ack! */
alt copy p.token {
ACTUALLY(token::w_expr(e)) {
p.bump();
ret pexpr(e);
}
ACTUALLY(token::w_path(pt)) {
p.bump();
ret p.mk_pexpr(p.span.lo, p.span.lo,
expr_path(pt));
}
_ {} }]];
#macro[[#maybe_whole_ty[p],
alt copy p.token {
ACTUALLY(token::w_ty(t)) { p.bump(); ret t; }
_ {} }]];
/* ident is handled by common.rs */
#macro[[#maybe_whole_path[p],
alt p.token {
ACTUALLY(token::w_path(pt)) { p.bump(); ret pt; }
_ {} }]];
}
class parser {
let sess: parse_sess;
let cfg: crate_cfg;
@ -718,6 +790,7 @@ class parser {
}
fn parse_bottom_expr() -> pexpr {
#maybe_whole_expr_pexpr[self];
let lo = self.span.lo;
let mut hi = self.span.hi;
@ -1181,6 +1254,7 @@ class parser {
fn parse_prefix_expr() -> pexpr {
#maybe_whole_expr_pexpr[self];
let lo = self.span.lo;
let mut hi;
@ -1258,6 +1332,7 @@ class parser {
fn parse_more_binops(plhs: pexpr, min_prec: uint) ->
@expr {
#maybe_whole_expr[self];
let lhs = self.to_expr(plhs);
if self.expr_is_complete(plhs) { ret lhs; }
let peeked = self.token;

2
src/libsyntax/parse/token.rs
View File

@ -94,7 +94,7 @@ enum whole_nt {
w_pat( @ast::pat),
w_expr(@ast::expr),
w_ty( @ast::ty),
w_ident(ast::ident),
w_ident(str_num, bool),
w_path(@ast::path),
}