remove peek_span_src_raw from StringReader
This commit is contained in:
parent
e9dc95c86e
commit
256df83f64
|
@ -53,7 +53,7 @@ impl<'a> SpanUtils<'a> {
|
||||||
pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
|
pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
|
||||||
let mut toks = self.retokenise_span(span);
|
let mut toks = self.retokenise_span(span);
|
||||||
loop {
|
loop {
|
||||||
let next = toks.real_token();
|
let next = toks.next_token();
|
||||||
if next == token::Eof {
|
if next == token::Eof {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,8 +38,6 @@ pub struct StringReader<'a> {
|
||||||
crate source_file: Lrc<syntax_pos::SourceFile>,
|
crate source_file: Lrc<syntax_pos::SourceFile>,
|
||||||
/// Stop reading src at this index.
|
/// Stop reading src at this index.
|
||||||
crate end_src_index: usize,
|
crate end_src_index: usize,
|
||||||
// cached:
|
|
||||||
peek_span_src_raw: Span,
|
|
||||||
fatal_errs: Vec<DiagnosticBuilder<'a>>,
|
fatal_errs: Vec<DiagnosticBuilder<'a>>,
|
||||||
// cache a direct reference to the source text, so that we don't have to
|
// cache a direct reference to the source text, so that we don't have to
|
||||||
// retrieve it via `self.source_file.src.as_ref().unwrap()` all the time.
|
// retrieve it via `self.source_file.src.as_ref().unwrap()` all the time.
|
||||||
|
@ -59,7 +57,7 @@ impl<'a> StringReader<'a> {
|
||||||
(real, raw)
|
(real, raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn unwrap_or_abort(&mut self, res: Result<Token, ()>) -> Token {
|
fn unwrap_or_abort<T>(&mut self, res: Result<T, ()>) -> T {
|
||||||
match res {
|
match res {
|
||||||
Ok(tok) => tok,
|
Ok(tok) => tok,
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
|
@ -69,36 +67,52 @@ impl<'a> StringReader<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_token(&mut self) -> Token where Self: Sized {
|
/// Returns the next token. EFFECT: advances the string_reader.
|
||||||
|
pub fn try_next_token(&mut self) -> Result<Token, ()> {
|
||||||
|
let (token, _raw_span) = self.try_next_token_with_raw_span()?;
|
||||||
|
Ok(token)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn next_token(&mut self) -> Token {
|
||||||
let res = self.try_next_token();
|
let res = self.try_next_token();
|
||||||
self.unwrap_or_abort(res)
|
self.unwrap_or_abort(res)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the next token. EFFECT: advances the string_reader.
|
fn try_real_token(&mut self) -> Result<(Token, Span), ()> {
|
||||||
pub fn try_next_token(&mut self) -> Result<Token, ()> {
|
|
||||||
assert!(self.fatal_errs.is_empty());
|
|
||||||
self.advance_token()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn try_real_token(&mut self) -> Result<Token, ()> {
|
|
||||||
let mut t = self.try_next_token()?;
|
|
||||||
loop {
|
loop {
|
||||||
match t.kind {
|
let t = self.try_next_token_with_raw_span()?;
|
||||||
token::Whitespace | token::Comment | token::Shebang(_) => {
|
match t.0.kind {
|
||||||
t = self.try_next_token()?;
|
token::Whitespace | token::Comment | token::Shebang(_) => continue,
|
||||||
}
|
_ => return Ok(t),
|
||||||
_ => break,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn real_token(&mut self) -> Token {
|
fn real_token(&mut self) -> (Token, Span) {
|
||||||
let res = self.try_real_token();
|
let res = self.try_real_token();
|
||||||
self.unwrap_or_abort(res)
|
self.unwrap_or_abort(res)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn try_next_token_with_raw_span(&mut self) -> Result<(Token, Span), ()> {
|
||||||
|
assert!(self.fatal_errs.is_empty());
|
||||||
|
match self.scan_whitespace_or_comment() {
|
||||||
|
Some(comment) => {
|
||||||
|
let raw_span = comment.span;
|
||||||
|
Ok((comment, raw_span))
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
let (kind, start_pos, end_pos) = if self.is_eof() {
|
||||||
|
(token::Eof, self.source_file.end_pos, self.source_file.end_pos)
|
||||||
|
} else {
|
||||||
|
let start_pos = self.pos;
|
||||||
|
(self.next_token_inner()?, start_pos, self.pos)
|
||||||
|
};
|
||||||
|
let (real, raw) = self.mk_sp_and_raw(start_pos, end_pos);
|
||||||
|
Ok((Token::new(kind, real), raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn is_eof(&self) -> bool {
|
fn is_eof(&self) -> bool {
|
||||||
self.ch.is_none()
|
self.ch.is_none()
|
||||||
|
@ -141,7 +155,6 @@ impl<'a> StringReader<'a> {
|
||||||
override_span: Option<Span>) -> Self {
|
override_span: Option<Span>) -> Self {
|
||||||
let mut sr = StringReader::new_raw_internal(sess, source_file, override_span);
|
let mut sr = StringReader::new_raw_internal(sess, source_file, override_span);
|
||||||
sr.bump();
|
sr.bump();
|
||||||
|
|
||||||
sr
|
sr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -162,7 +175,6 @@ impl<'a> StringReader<'a> {
|
||||||
ch: Some('\n'),
|
ch: Some('\n'),
|
||||||
source_file,
|
source_file,
|
||||||
end_src_index: src.len(),
|
end_src_index: src.len(),
|
||||||
peek_span_src_raw: syntax_pos::DUMMY_SP,
|
|
||||||
src,
|
src,
|
||||||
fatal_errs: Vec::new(),
|
fatal_errs: Vec::new(),
|
||||||
override_span,
|
override_span,
|
||||||
|
@ -172,12 +184,8 @@ impl<'a> StringReader<'a> {
|
||||||
pub fn new_or_buffered_errs(sess: &'a ParseSess,
|
pub fn new_or_buffered_errs(sess: &'a ParseSess,
|
||||||
source_file: Lrc<syntax_pos::SourceFile>,
|
source_file: Lrc<syntax_pos::SourceFile>,
|
||||||
override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
|
override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
|
||||||
let mut sr = StringReader::new_raw(sess, source_file, override_span);
|
let sr = StringReader::new_raw(sess, source_file, override_span);
|
||||||
if sr.advance_token().is_err() {
|
Ok(sr)
|
||||||
Err(sr.buffer_fatal_errors())
|
|
||||||
} else {
|
|
||||||
Ok(sr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
|
pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
|
||||||
|
@ -197,11 +205,6 @@ impl<'a> StringReader<'a> {
|
||||||
|
|
||||||
sr.bump();
|
sr.bump();
|
||||||
|
|
||||||
if sr.advance_token().is_err() {
|
|
||||||
sr.emit_fatal_errors();
|
|
||||||
FatalError.raise();
|
|
||||||
}
|
|
||||||
|
|
||||||
sr
|
sr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -257,28 +260,6 @@ impl<'a> StringReader<'a> {
|
||||||
self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), &m[..])
|
self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), &m[..])
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Advance peek_token to refer to the next token, and
|
|
||||||
/// possibly update the interner.
|
|
||||||
fn advance_token(&mut self) -> Result<Token, ()> {
|
|
||||||
match self.scan_whitespace_or_comment() {
|
|
||||||
Some(comment) => {
|
|
||||||
self.peek_span_src_raw = comment.span;
|
|
||||||
Ok(comment)
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
let (kind, start_pos, end_pos) = if self.is_eof() {
|
|
||||||
(token::Eof, self.source_file.end_pos, self.source_file.end_pos)
|
|
||||||
} else {
|
|
||||||
let start_pos = self.pos;
|
|
||||||
(self.next_token_inner()?, start_pos, self.pos)
|
|
||||||
};
|
|
||||||
let (real, raw) = self.mk_sp_and_raw(start_pos, end_pos);
|
|
||||||
self.peek_span_src_raw = raw;
|
|
||||||
Ok(Token::new(kind, real))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn src_index(&self, pos: BytePos) -> usize {
|
fn src_index(&self, pos: BytePos) -> usize {
|
||||||
(pos - self.source_file.start_pos).to_usize()
|
(pos - self.source_file.start_pos).to_usize()
|
||||||
|
@ -1447,12 +1428,7 @@ mod tests {
|
||||||
teststr: String)
|
teststr: String)
|
||||||
-> StringReader<'a> {
|
-> StringReader<'a> {
|
||||||
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
|
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
|
||||||
let mut sr = StringReader::new_raw(sess, sf, None);
|
StringReader::new_raw(sess, sf, None)
|
||||||
if sr.advance_token().is_err() {
|
|
||||||
sr.emit_fatal_errors();
|
|
||||||
FatalError.raise();
|
|
||||||
}
|
|
||||||
sr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use syntax_pos::Span;
|
use syntax_pos::{Span, DUMMY_SP};
|
||||||
|
|
||||||
use crate::print::pprust::token_to_string;
|
use crate::print::pprust::token_to_string;
|
||||||
use crate::parse::lexer::{StringReader, UnmatchedBrace};
|
use crate::parse::lexer::{StringReader, UnmatchedBrace};
|
||||||
|
@ -11,6 +11,7 @@ impl<'a> StringReader<'a> {
|
||||||
let mut tt_reader = TokenTreesReader {
|
let mut tt_reader = TokenTreesReader {
|
||||||
string_reader: self,
|
string_reader: self,
|
||||||
token: Token::dummy(),
|
token: Token::dummy(),
|
||||||
|
raw_span: DUMMY_SP,
|
||||||
open_braces: Vec::new(),
|
open_braces: Vec::new(),
|
||||||
unmatched_braces: Vec::new(),
|
unmatched_braces: Vec::new(),
|
||||||
matching_delim_spans: Vec::new(),
|
matching_delim_spans: Vec::new(),
|
||||||
|
@ -24,6 +25,7 @@ impl<'a> StringReader<'a> {
|
||||||
struct TokenTreesReader<'a> {
|
struct TokenTreesReader<'a> {
|
||||||
string_reader: StringReader<'a>,
|
string_reader: StringReader<'a>,
|
||||||
token: Token,
|
token: Token,
|
||||||
|
raw_span: Span,
|
||||||
/// Stack of open delimiters and their spans. Used for error message.
|
/// Stack of open delimiters and their spans. Used for error message.
|
||||||
open_braces: Vec<(token::DelimToken, Span)>,
|
open_braces: Vec<(token::DelimToken, Span)>,
|
||||||
unmatched_braces: Vec<UnmatchedBrace>,
|
unmatched_braces: Vec<UnmatchedBrace>,
|
||||||
|
@ -206,18 +208,17 @@ impl<'a> TokenTreesReader<'a> {
|
||||||
// Note that testing for joint-ness here is done via the raw
|
// Note that testing for joint-ness here is done via the raw
|
||||||
// source span as the joint-ness is a property of the raw source
|
// source span as the joint-ness is a property of the raw source
|
||||||
// rather than wanting to take `override_span` into account.
|
// rather than wanting to take `override_span` into account.
|
||||||
// Additionally, we actually check if the *next* pair of tokens
|
let raw_span = self.raw_span;
|
||||||
// is joint, but this is equivalent to checking the current pair.
|
|
||||||
let raw = self.string_reader.peek_span_src_raw;
|
|
||||||
self.real_token();
|
self.real_token();
|
||||||
let is_joint = raw.hi() == self.string_reader.peek_span_src_raw.lo()
|
let is_joint = raw_span.hi() == self.raw_span.lo() && self.token.is_op();
|
||||||
&& self.token.is_op();
|
|
||||||
Ok((tt, if is_joint { Joint } else { NonJoint }))
|
Ok((tt, if is_joint { Joint } else { NonJoint }))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn real_token(&mut self) {
|
fn real_token(&mut self) {
|
||||||
self.token = self.string_reader.real_token();
|
let (token, raw_span) = self.string_reader.real_token();
|
||||||
|
self.token = token;
|
||||||
|
self.raw_span = raw_span;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue