cleanup lexer constructors
This commit is contained in:
parent
256df83f64
commit
601bad86b2
|
@ -38,17 +38,17 @@ pub fn render_with_highlighting(
|
|||
FileName::Custom(String::from("rustdoc-highlighting")),
|
||||
src.to_owned(),
|
||||
);
|
||||
let highlight_result =
|
||||
lexer::StringReader::new_or_buffered_errs(&sess, fm, None).and_then(|lexer| {
|
||||
let mut classifier = Classifier::new(lexer, sess.source_map());
|
||||
let highlight_result = {
|
||||
let lexer = lexer::StringReader::new(&sess, fm, None);
|
||||
let mut classifier = Classifier::new(lexer, sess.source_map());
|
||||
|
||||
let mut highlighted_source = vec![];
|
||||
if classifier.write_source(&mut highlighted_source).is_err() {
|
||||
Err(classifier.lexer.buffer_fatal_errors())
|
||||
} else {
|
||||
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
|
||||
}
|
||||
});
|
||||
let mut highlighted_source = vec![];
|
||||
if classifier.write_source(&mut highlighted_source).is_err() {
|
||||
Err(classifier.lexer.buffer_fatal_errors())
|
||||
} else {
|
||||
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
|
||||
}
|
||||
};
|
||||
|
||||
match highlight_result {
|
||||
Ok(highlighted_source) => {
|
||||
|
|
|
@ -32,7 +32,8 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
|
|||
dox[code_block.code].to_owned(),
|
||||
);
|
||||
|
||||
let errors = Lexer::new_or_buffered_errs(&sess, source_file, None).and_then(|mut lexer| {
|
||||
let errors = {
|
||||
let mut lexer = Lexer::new(&sess, source_file, None);
|
||||
while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
|
||||
if kind == token::Eof {
|
||||
break;
|
||||
|
@ -46,7 +47,7 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
|
|||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
if let Err(errors) = errors {
|
||||
let mut diag = if let Some(sp) =
|
||||
|
|
|
@ -346,7 +346,7 @@ pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) ->
|
|||
srdr.read_to_string(&mut src).unwrap();
|
||||
let cm = SourceMap::new(sess.source_map().path_mapping().clone());
|
||||
let source_file = cm.new_source_file(path, src);
|
||||
let mut rdr = lexer::StringReader::new_raw(sess, source_file, None);
|
||||
let mut rdr = lexer::StringReader::new(sess, source_file, None);
|
||||
|
||||
let mut comments: Vec<Comment> = Vec::new();
|
||||
let mut code_to_the_left = false; // Only code
|
||||
|
|
|
@ -149,16 +149,15 @@ impl<'a> StringReader<'a> {
|
|||
buffer
|
||||
}
|
||||
|
||||
/// For comments.rs, which hackily pokes into next_pos and ch
|
||||
fn new_raw(sess: &'a ParseSess,
|
||||
pub fn new(sess: &'a ParseSess,
|
||||
source_file: Lrc<syntax_pos::SourceFile>,
|
||||
override_span: Option<Span>) -> Self {
|
||||
let mut sr = StringReader::new_raw_internal(sess, source_file, override_span);
|
||||
let mut sr = StringReader::new_internal(sess, source_file, override_span);
|
||||
sr.bump();
|
||||
sr
|
||||
}
|
||||
|
||||
fn new_raw_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>,
|
||||
fn new_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>,
|
||||
override_span: Option<Span>) -> Self
|
||||
{
|
||||
if source_file.src.is_none() {
|
||||
|
@ -181,13 +180,6 @@ impl<'a> StringReader<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn new_or_buffered_errs(sess: &'a ParseSess,
|
||||
source_file: Lrc<syntax_pos::SourceFile>,
|
||||
override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
|
||||
let sr = StringReader::new_raw(sess, source_file, override_span);
|
||||
Ok(sr)
|
||||
}
|
||||
|
||||
pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
|
||||
let begin = sess.source_map().lookup_byte_offset(span.lo());
|
||||
let end = sess.source_map().lookup_byte_offset(span.hi());
|
||||
|
@ -197,7 +189,7 @@ impl<'a> StringReader<'a> {
|
|||
span = span.shrink_to_lo();
|
||||
}
|
||||
|
||||
let mut sr = StringReader::new_raw_internal(sess, begin.sf, None);
|
||||
let mut sr = StringReader::new_internal(sess, begin.sf, None);
|
||||
|
||||
// Seek the lexer to the right byte range.
|
||||
sr.next_pos = span.lo();
|
||||
|
@ -1428,7 +1420,7 @@ mod tests {
|
|||
teststr: String)
|
||||
-> StringReader<'a> {
|
||||
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
|
||||
StringReader::new_raw(sess, sf, None)
|
||||
StringReader::new(sess, sf, None)
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
@ -305,7 +305,7 @@ pub fn maybe_file_to_stream(
|
|||
source_file: Lrc<SourceFile>,
|
||||
override_span: Option<Span>,
|
||||
) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> {
|
||||
let srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?;
|
||||
let srdr = lexer::StringReader::new(sess, source_file, override_span);
|
||||
let (token_trees, unmatched_braces) = srdr.into_token_trees();
|
||||
|
||||
match token_trees {
|
||||
|
|
Loading…
Reference in New Issue