OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

auto merge of #14630 : cmr/rust/rewrite-lexer, r=alexcrichton 

These are a pain to rebase, so I'm separating this from the rest of my work.
Nothing controversial here, just some simple refactoring and removal of an
unused entry in the token table. Brings the lexer into 2012 with methods!
This commit is contained in:
bors 2014-06-04 13:06:47 -07:00
parent 7645982efc 181e5f3fc8
commit ef9bf3a4ee
9 changed files with 1216 additions and 1204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/librustdoc/html/highlight.rs
View File

@ -34,7 +34,7 @@ pub fn highlight(src: &str, class: Option<&str>) -> String {
let mut out = io::MemWriter::new();
doit(&sess,
lexer::new_string_reader(&sess.span_diagnostic, fm),
lexer::StringReader::new(&sess.span_diagnostic, fm),
class,
&mut out).unwrap();
str::from_utf8_lossy(out.unwrap().as_slice()).to_string()

2
src/libsyntax/attr.rs
View File

@ -15,7 +15,7 @@ use ast::{AttrId, Attribute, Attribute_, MetaItem, MetaWord, MetaNameValue, Meta
use codemap::{Span, Spanned, spanned, dummy_spanned};
use codemap::BytePos;
use diagnostic::SpanHandler;
use parse::comments::{doc_comment_style, strip_doc_comment_decoration};
use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
use parse::token::InternedString;
use parse::token;
use crateid::CrateId;

1112
src/libsyntax/parse/lexer.rs
View File

File diff suppressed because it is too large Load Diff

105
src/libsyntax/parse/comments.rs → src/libsyntax/parse/lexer/comments.rs
View File

@ -11,8 +11,8 @@
use ast;
use codemap::{BytePos, CharPos, CodeMap, Pos};
use diagnostic;
use parse::lexer::{is_whitespace, with_str_from, Reader};
use parse::lexer::{StringReader, bump, is_eof, nextch_is, TokenAndSpan};
use parse::lexer::{is_whitespace, Reader};
use parse::lexer::{StringReader, TokenAndSpan};
use parse::lexer::{is_line_non_doc_comment, is_block_non_doc_comment};
use parse::lexer;
use parse::token;
@ -141,31 +141,6 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String {
fail!("not a doc-comment: {}", comment);
}
fn read_to_eol(rdr: &mut StringReader) -> String {
let mut val = String::new();
while !rdr.curr_is('\n') && !is_eof(rdr) {
val.push_char(rdr.curr.unwrap());
bump(rdr);
}
if rdr.curr_is('\n') { bump(rdr); }
return val
}
fn read_one_line_comment(rdr: &mut StringReader) -> String {
let val = read_to_eol(rdr);
assert!((val.as_slice()[0] == '/' as u8 &&
val.as_slice()[1] == '/' as u8) ||
(val.as_slice()[0] == '#' as u8 &&
val.as_slice()[1] == '!' as u8));
return val;
}
fn consume_non_eol_whitespace(rdr: &mut StringReader) {
while is_whitespace(rdr.curr) && !rdr.curr_is('\n') && !is_eof(rdr) {
bump(rdr);
}
}
fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) {
debug!(">>> blank-line comment");
comments.push(Comment {
@ -177,11 +152,11 @@ fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) {
fn consume_whitespace_counting_blank_lines(rdr: &mut StringReader,
comments: &mut Vec<Comment>) {
while is_whitespace(rdr.curr) && !is_eof(rdr) {
while is_whitespace(rdr.curr) && !rdr.is_eof() {
if rdr.col == CharPos(0u) && rdr.curr_is('\n') {
push_blank_line_comment(rdr, &mut *comments);
}
bump(rdr);
rdr.bump();
}
}
@ -193,7 +168,7 @@ fn read_shebang_comment(rdr: &mut StringReader, code_to_the_left: bool,
debug!("<<< shebang comment");
comments.push(Comment {
style: if code_to_the_left { Trailing } else { Isolated },
lines: vec!(read_one_line_comment(rdr)),
lines: vec!(rdr.read_one_line_comment()),
pos: p
});
}
@ -203,15 +178,15 @@ fn read_line_comments(rdr: &mut StringReader, code_to_the_left: bool,
debug!(">>> line comments");
let p = rdr.last_pos;
let mut lines: Vec<String> = Vec::new();
while rdr.curr_is('/') && nextch_is(rdr, '/') {
let line = read_one_line_comment(rdr);
while rdr.curr_is('/') && rdr.nextch_is('/') {
let line = rdr.read_one_line_comment();
debug!("{}", line);
// Doc comments are not put in comments.
if is_doc_comment(line.as_slice()) {
break;
}
lines.push(line);
consume_non_eol_whitespace(rdr);
rdr.consume_non_eol_whitespace();
}
debug!("<<< line comments");
if !lines.is_empty() {
@ -265,21 +240,21 @@ fn read_block_comment(rdr: &mut StringReader,
let p = rdr.last_pos;
let mut lines: Vec<String> = Vec::new();
let col = rdr.col;
bump(rdr);
bump(rdr);
rdr.bump();
rdr.bump();
let mut curr_line = String::from_str("/*");
// doc-comments are not really comments, they are attributes
if (rdr.curr_is('*') && !nextch_is(rdr, '*')) || rdr.curr_is('!') {
while !(rdr.curr_is('*') && nextch_is(rdr, '/')) && !is_eof(rdr) {
if (rdr.curr_is('*') && !rdr.nextch_is('*')) || rdr.curr_is('!') {
while !(rdr.curr_is('*') && rdr.nextch_is('/')) && !rdr.is_eof() {
curr_line.push_char(rdr.curr.unwrap());
bump(rdr);
rdr.bump();
}
if !is_eof(rdr) {
if !rdr.is_eof() {
curr_line.push_str("*/");
bump(rdr);
bump(rdr);
rdr.bump();
rdr.bump();
}
if !is_block_non_doc_comment(curr_line.as_slice()) {
return
@ -290,7 +265,7 @@ fn read_block_comment(rdr: &mut StringReader,
let mut level: int = 1;
while level > 0 {
debug!("=== block comment level {}", level);
if is_eof(rdr) {
if rdr.is_eof() {
rdr.fatal("unterminated block comment");
}
if rdr.curr_is('\n') {
@ -298,21 +273,21 @@ fn read_block_comment(rdr: &mut StringReader,
curr_line,
col);
curr_line = String::new();
bump(rdr);
rdr.bump();
} else {
curr_line.push_char(rdr.curr.unwrap());
if rdr.curr_is('/') && nextch_is(rdr, '*') {
bump(rdr);
bump(rdr);
if rdr.curr_is('/') && rdr.nextch_is('*') {
rdr.bump();
rdr.bump();
curr_line.push_char('*');
level += 1;
} else {
if rdr.curr_is('*') && nextch_is(rdr, '/') {
bump(rdr);
bump(rdr);
if rdr.curr_is('*') && rdr.nextch_is('/') {
rdr.bump();
rdr.bump();
curr_line.push_char('/');
level -= 1;
} else { bump(rdr); }
} else { rdr.bump(); }
}
}
}
@ -324,31 +299,24 @@ fn read_block_comment(rdr: &mut StringReader,
}
let mut style = if code_to_the_left { Trailing } else { Isolated };
consume_non_eol_whitespace(rdr);
if !is_eof(rdr) && !rdr.curr_is('\n') && lines.len() == 1u {
rdr.consume_non_eol_whitespace();
if !rdr.is_eof() && !rdr.curr_is('\n') && lines.len() == 1u {
style = Mixed;
}
debug!("<<< block comment");
comments.push(Comment {style: style, lines: lines, pos: p});
}
fn peeking_at_comment(rdr: &StringReader) -> bool {
return (rdr.curr_is('/') && nextch_is(rdr, '/')) ||
(rdr.curr_is('/') && nextch_is(rdr, '*')) ||
// consider shebangs comments, but not inner attributes
(rdr.curr_is('#') && nextch_is(rdr, '!') &&
!lexer::nextnextch_is(rdr, '['));
}
fn consume_comment(rdr: &mut StringReader,
code_to_the_left: bool,
comments: &mut Vec<Comment> ) {
debug!(">>> consume comment");
if rdr.curr_is('/') && nextch_is(rdr, '/') {
if rdr.curr_is('/') && rdr.nextch_is('/') {
read_line_comments(rdr, code_to_the_left, comments);
} else if rdr.curr_is('/') && nextch_is(rdr, '*') {
} else if rdr.curr_is('/') && rdr.nextch_is('*') {
read_block_comment(rdr, code_to_the_left, comments);
} else if rdr.curr_is('#') && nextch_is(rdr, '!') {
} else if rdr.curr_is('#') && rdr.nextch_is('!') {
read_shebang_comment(rdr, code_to_the_left, comments);
} else { fail!(); }
debug!("<<< consume comment");
@ -362,8 +330,7 @@ pub struct Literal {
// it appears this function is called only from pprust... that's
// probably not a good thing.
pub fn gather_comments_and_literals(span_diagnostic:
&diagnostic::SpanHandler,
pub fn gather_comments_and_literals(span_diagnostic: &diagnostic::SpanHandler,
path: String,
srdr: &mut io::Reader)
-> (Vec<Comment>, Vec<Literal>) {
@ -371,20 +338,20 @@ pub fn gather_comments_and_literals(span_diagnostic:
let src = str::from_utf8(src.as_slice()).unwrap().to_string();
let cm = CodeMap::new();
let filemap = cm.new_filemap(path, src);
let mut rdr = lexer::new_low_level_string_reader(span_diagnostic, filemap);
let mut rdr = lexer::StringReader::new_raw(span_diagnostic, filemap);
let mut comments: Vec<Comment> = Vec::new();
let mut literals: Vec<Literal> = Vec::new();
let mut first_read: bool = true;
while !is_eof(&rdr) {
while !rdr.is_eof() {
loop {
let mut code_to_the_left = !first_read;
consume_non_eol_whitespace(&mut rdr);
rdr.consume_non_eol_whitespace();
if rdr.curr_is('\n') {
code_to_the_left = false;
consume_whitespace_counting_blank_lines(&mut rdr, &mut comments);
}
while peeking_at_comment(&rdr) {
while rdr.peeking_at_comment() {
consume_comment(&mut rdr, code_to_the_left, &mut comments);
consume_whitespace_counting_blank_lines(&mut rdr, &mut comments);
}
@ -397,7 +364,7 @@ pub fn gather_comments_and_literals(span_diagnostic:
//discard, and look ahead; we're working with internal state
let TokenAndSpan {tok: tok, sp: sp} = rdr.peek();
if token::is_lit(&tok) {
with_str_from(&rdr, bstart, |s| {
rdr.with_str_from(bstart, |s| {
debug!("tok lit: {}", s);
literals.push(Literal {lit: s.to_string(), pos: sp.lo});
})

1153
src/libsyntax/parse/lexer/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

3
src/libsyntax/parse/mod.rs
View File

@ -25,7 +25,6 @@ use std::str;
pub mod lexer;
pub mod parser;
pub mod token;
pub mod comments;
pub mod attr;
pub mod common;
@ -255,7 +254,7 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>)
// it appears to me that the cfg doesn't matter here... indeed,
// parsing tt's probably shouldn't require a parser at all.
let cfg = Vec::new();
let srdr = lexer::new_string_reader(&sess.span_diagnostic, filemap);
let srdr = lexer::StringReader::new(&sess.span_diagnostic, filemap);
let mut p1 = Parser::new(sess, cfg, box srdr);
p1.parse_all_token_trees()
}

30
src/libsyntax/parse/token.rs
View File

@ -136,18 +136,18 @@ impl fmt::Show for Nonterminal {
}
}
pub fn binop_to_str(o: BinOp) -> String {
pub fn binop_to_str(o: BinOp) -> &'static str {
match o {
PLUS => "+".to_string(),
MINUS => "-".to_string(),
STAR => "*".to_string(),
SLASH => "/".to_string(),
PERCENT => "%".to_string(),
CARET => "^".to_string(),
AND => "&".to_string(),
OR => "|".to_string(),
SHL => "<<".to_string(),
SHR => ">>".to_string()
PLUS => "+",
MINUS => "-",
STAR => "*",
SLASH => "/",
PERCENT => "%",
CARET => "^",
AND => "&",
OR => "|",
SHL => "<<",
SHR => ">>"
}
}
@ -164,9 +164,9 @@ pub fn to_str(t: &Token) -> String {
TILDE => "~".to_string(),
OROR => "||".to_string(),
ANDAND => "&&".to_string(),
BINOP(op) => binop_to_str(op),
BINOP(op) => binop_to_str(op).to_string(),
BINOPEQ(op) => {
let mut s = binop_to_str(op);
let mut s = binop_to_str(op).to_string();
s.push_str("=");
s
}
@ -423,6 +423,10 @@ macro_rules! declare_special_idents_and_keywords {(
static SELF_KEYWORD_NAME: Name = 1;
static STATIC_KEYWORD_NAME: Name = 2;
// NB: leaving holes in the ident table is bad! a different ident will get
// interned with the id from the hole, but it will be between the min and max
// of the reserved words, and thus tagged as "reserved".
declare_special_idents_and_keywords! {
pub mod special_idents {
// These ones are statics

3
src/libsyntax/print/pprust.rs
View File

@ -20,7 +20,8 @@ use codemap;
use diagnostic;
use parse::classify::expr_is_simple_block;
use parse::token::IdentInterner;
use parse::{comments, token};
use parse::token;
use parse::lexer::comments;
use parse;
use print::pp::{break_offset, word, space, zerobreak, hardbreak};
use print::pp::{Breaks, Consistent, Inconsistent, eof};

10
src/libsyntax/util/interner.rs
View File

@ -126,14 +126,14 @@ impl RcStr {
}
}
// A StrInterner differs from Interner<String> in that it accepts
// &str rather than RcStr, resulting in less allocation.
/// A StrInterner differs from Interner<String> in that it accepts
/// &str rather than RcStr, resulting in less allocation.
pub struct StrInterner {
map: RefCell<HashMap<RcStr, Name>>,
vect: RefCell<Vec<RcStr> >,
}
// when traits can extend traits, we should extend index<Name,T> to get []
/// When traits can extend traits, we should extend index<Name,T> to get []
impl StrInterner {
pub fn new() -> StrInterner {
StrInterner {
@ -177,8 +177,8 @@ impl StrInterner {
// lightweight way to get what I want, though not
// necessarily the cleanest.
// create a gensym with the same name as an existing
// entry.
/// Create a gensym with the same name as an existing
/// entry.
pub fn gensym_copy(&self, idx : Name) -> Name {
let new_idx = self.len() as Name;
// leave out of map to avoid colliding