cleanup lexer constructors
This commit is contained in:
parent
256df83f64
commit
601bad86b2
|
@ -38,17 +38,17 @@ pub fn render_with_highlighting(
|
||||||
FileName::Custom(String::from("rustdoc-highlighting")),
|
FileName::Custom(String::from("rustdoc-highlighting")),
|
||||||
src.to_owned(),
|
src.to_owned(),
|
||||||
);
|
);
|
||||||
let highlight_result =
|
let highlight_result = {
|
||||||
lexer::StringReader::new_or_buffered_errs(&sess, fm, None).and_then(|lexer| {
|
let lexer = lexer::StringReader::new(&sess, fm, None);
|
||||||
let mut classifier = Classifier::new(lexer, sess.source_map());
|
let mut classifier = Classifier::new(lexer, sess.source_map());
|
||||||
|
|
||||||
let mut highlighted_source = vec![];
|
let mut highlighted_source = vec![];
|
||||||
if classifier.write_source(&mut highlighted_source).is_err() {
|
if classifier.write_source(&mut highlighted_source).is_err() {
|
||||||
Err(classifier.lexer.buffer_fatal_errors())
|
Err(classifier.lexer.buffer_fatal_errors())
|
||||||
} else {
|
} else {
|
||||||
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
|
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
|
||||||
}
|
}
|
||||||
});
|
};
|
||||||
|
|
||||||
match highlight_result {
|
match highlight_result {
|
||||||
Ok(highlighted_source) => {
|
Ok(highlighted_source) => {
|
||||||
|
|
|
@ -32,7 +32,8 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
|
||||||
dox[code_block.code].to_owned(),
|
dox[code_block.code].to_owned(),
|
||||||
);
|
);
|
||||||
|
|
||||||
let errors = Lexer::new_or_buffered_errs(&sess, source_file, None).and_then(|mut lexer| {
|
let errors = {
|
||||||
|
let mut lexer = Lexer::new(&sess, source_file, None);
|
||||||
while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
|
while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
|
||||||
if kind == token::Eof {
|
if kind == token::Eof {
|
||||||
break;
|
break;
|
||||||
|
@ -46,7 +47,7 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
});
|
};
|
||||||
|
|
||||||
if let Err(errors) = errors {
|
if let Err(errors) = errors {
|
||||||
let mut diag = if let Some(sp) =
|
let mut diag = if let Some(sp) =
|
||||||
|
|
|
@ -346,7 +346,7 @@ pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) ->
|
||||||
srdr.read_to_string(&mut src).unwrap();
|
srdr.read_to_string(&mut src).unwrap();
|
||||||
let cm = SourceMap::new(sess.source_map().path_mapping().clone());
|
let cm = SourceMap::new(sess.source_map().path_mapping().clone());
|
||||||
let source_file = cm.new_source_file(path, src);
|
let source_file = cm.new_source_file(path, src);
|
||||||
let mut rdr = lexer::StringReader::new_raw(sess, source_file, None);
|
let mut rdr = lexer::StringReader::new(sess, source_file, None);
|
||||||
|
|
||||||
let mut comments: Vec<Comment> = Vec::new();
|
let mut comments: Vec<Comment> = Vec::new();
|
||||||
let mut code_to_the_left = false; // Only code
|
let mut code_to_the_left = false; // Only code
|
||||||
|
|
|
@ -149,16 +149,15 @@ impl<'a> StringReader<'a> {
|
||||||
buffer
|
buffer
|
||||||
}
|
}
|
||||||
|
|
||||||
/// For comments.rs, which hackily pokes into next_pos and ch
|
pub fn new(sess: &'a ParseSess,
|
||||||
fn new_raw(sess: &'a ParseSess,
|
|
||||||
source_file: Lrc<syntax_pos::SourceFile>,
|
source_file: Lrc<syntax_pos::SourceFile>,
|
||||||
override_span: Option<Span>) -> Self {
|
override_span: Option<Span>) -> Self {
|
||||||
let mut sr = StringReader::new_raw_internal(sess, source_file, override_span);
|
let mut sr = StringReader::new_internal(sess, source_file, override_span);
|
||||||
sr.bump();
|
sr.bump();
|
||||||
sr
|
sr
|
||||||
}
|
}
|
||||||
|
|
||||||
fn new_raw_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>,
|
fn new_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>,
|
||||||
override_span: Option<Span>) -> Self
|
override_span: Option<Span>) -> Self
|
||||||
{
|
{
|
||||||
if source_file.src.is_none() {
|
if source_file.src.is_none() {
|
||||||
|
@ -181,13 +180,6 @@ impl<'a> StringReader<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_or_buffered_errs(sess: &'a ParseSess,
|
|
||||||
source_file: Lrc<syntax_pos::SourceFile>,
|
|
||||||
override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
|
|
||||||
let sr = StringReader::new_raw(sess, source_file, override_span);
|
|
||||||
Ok(sr)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
|
pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
|
||||||
let begin = sess.source_map().lookup_byte_offset(span.lo());
|
let begin = sess.source_map().lookup_byte_offset(span.lo());
|
||||||
let end = sess.source_map().lookup_byte_offset(span.hi());
|
let end = sess.source_map().lookup_byte_offset(span.hi());
|
||||||
|
@ -197,7 +189,7 @@ impl<'a> StringReader<'a> {
|
||||||
span = span.shrink_to_lo();
|
span = span.shrink_to_lo();
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut sr = StringReader::new_raw_internal(sess, begin.sf, None);
|
let mut sr = StringReader::new_internal(sess, begin.sf, None);
|
||||||
|
|
||||||
// Seek the lexer to the right byte range.
|
// Seek the lexer to the right byte range.
|
||||||
sr.next_pos = span.lo();
|
sr.next_pos = span.lo();
|
||||||
|
@ -1428,7 +1420,7 @@ mod tests {
|
||||||
teststr: String)
|
teststr: String)
|
||||||
-> StringReader<'a> {
|
-> StringReader<'a> {
|
||||||
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
|
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
|
||||||
StringReader::new_raw(sess, sf, None)
|
StringReader::new(sess, sf, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
@ -305,7 +305,7 @@ pub fn maybe_file_to_stream(
|
||||||
source_file: Lrc<SourceFile>,
|
source_file: Lrc<SourceFile>,
|
||||||
override_span: Option<Span>,
|
override_span: Option<Span>,
|
||||||
) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> {
|
) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> {
|
||||||
let srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?;
|
let srdr = lexer::StringReader::new(sess, source_file, override_span);
|
||||||
let (token_trees, unmatched_braces) = srdr.into_token_trees();
|
let (token_trees, unmatched_braces) = srdr.into_token_trees();
|
||||||
|
|
||||||
match token_trees {
|
match token_trees {
|
||||||
|
|
Loading…
Reference in New Issue