OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rollup merge of #47343 - goffrie:master, r=jseyfried

Browse Source 
Glued tokens can themselves be joint.

When gluing two tokens, the second of which is joint, the result should also be
joint.
This fixes an issue with joining three `Dot` tokens to make a `DotDotDot` - the
intermediate `DotDot` would not be joint and therefore we would not attempt to
glue the last `Dot` token, yielding `.. .` instead of `...`.

r? @jseyfried
This commit is contained in:
kennytm 2018-01-13 02:26:39 +08:00
commit b480366445
No known key found for this signature in database
GPG Key ID: FEF6C8051D0E013C
1 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/libsyntax/tokenstream.rs
View File

@ -286,12 +286,12 @@ impl TokenStream {
TokenStream::concat(result)
}
fn first_tree(&self) -> Option<TokenTree> {
fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
match self.kind {
TokenStreamKind::Empty => None,
TokenStreamKind::Tree(ref tree) |
TokenStreamKind::JointTree(ref tree) => Some(tree.clone()),
TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree(),
TokenStreamKind::Tree(ref tree) => Some((tree.clone(), false)),
TokenStreamKind::JointTree(ref tree) => Some((tree.clone(), true)),
TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
}
}
@ -315,12 +315,18 @@ impl TokenStreamBuilder {
let stream = stream.into();
let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
if let Some(TokenTree::Token(last_span, last_tok)) = last_tree_if_joint {
if let Some(TokenTree::Token(span, tok)) = stream.first_tree() {
if let Some((TokenTree::Token(span, tok), is_joint)) = stream.first_tree_and_joint() {
if let Some(glued_tok) = last_tok.glue(tok) {
let last_stream = self.0.pop().unwrap();
self.push_all_but_last_tree(&last_stream);
let glued_span = last_span.to(span);
self.0.push(TokenTree::Token(glued_span, glued_tok).into());
let glued_tt = TokenTree::Token(glued_span, glued_tok);
let glued_tokenstream = if is_joint {
glued_tt.joint()
} else {
glued_tt.into()
};
self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream);
return
}
@ -669,4 +675,16 @@ mod tests {
assert_eq!(test1.is_empty(), false);
assert_eq!(test2.is_empty(), false);
}
#[test]
fn test_dotdotdot() {
let mut builder = TokenStreamBuilder::new();
builder.push(TokenTree::Token(sp(0, 1), Token::Dot).joint());
builder.push(TokenTree::Token(sp(1, 2), Token::Dot).joint());
builder.push(TokenTree::Token(sp(2, 3), Token::Dot));
let stream = builder.build();
assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1);
}
}