From c2e26121dbf6f2ca08e772415b80bc2d80c0d952 Mon Sep 17 00:00:00 2001
From: James Mantooth <jimmantooth@gmail.com>
Date: Wed, 17 Jan 2018 05:23:21 -0600
Subject: [PATCH 01/26] Punctuation and clarity fixes.

---
 src/doc/unstable-book/src/language-features/generators.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/doc/unstable-book/src/language-features/generators.md b/src/doc/unstable-book/src/language-features/generators.md
index 7a559a7bec8..e8e2132dca2 100644
--- a/src/doc/unstable-book/src/language-features/generators.md
+++ b/src/doc/unstable-book/src/language-features/generators.md
@@ -139,11 +139,11 @@ closure-like semantics. Namely:
   types and such.
 
 * Traits like `Send` and `Sync` are automatically implemented for a `Generator`
-  depending on the captured variables of the environment. Unlike closures though
+  depending on the captured variables of the environment. Unlike closures,
   generators also depend on variables live across suspension points. This means
   that although the ambient environment may be `Send` or `Sync`, the generator
   itself may not be due to internal variables live across `yield` points being
-  not-`Send` or not-`Sync`. Note, though, that generators, like closures, do
+  not-`Send` or not-`Sync`. Note that generators, like closures, do
   not implement traits like `Copy` or `Clone` automatically.
 
 * Whenever a generator is dropped it will drop all captured environment
@@ -155,7 +155,7 @@ lifted at a future date, the design is ongoing!
 
 ### Generators as state machines
 
-In the compiler generators are currently compiled as state machines. Each
+In the compiler, generators are currently compiled as state machines. Each
 `yield` expression will correspond to a different state that stores all live
 variables over that suspension point. Resumption of a generator will dispatch on
 the current state and then execute internally until a `yield` is reached, at

From c4befe1710b3c394018ca65a6e99e109d081f16e Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Fri, 19 Jan 2018 15:46:15 -0600
Subject: [PATCH 02/26] Run rustfmt and add comments

---
 src/libsyntax/ext/tt/quoted.rs | 141 ++++++++++++++++++++++++---------
 1 file changed, 103 insertions(+), 38 deletions(-)

diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 0e21e3f6b00..ee87a612345 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -10,10 +10,10 @@
 
 use ast;
 use ext::tt::macro_parser;
-use parse::{ParseSess, token};
+use parse::{token, ParseSess};
 use print::pprust;
 use symbol::keywords;
-use syntax_pos::{DUMMY_SP, Span, BytePos};
+use syntax_pos::{BytePos, Span, DUMMY_SP};
 use tokenstream;
 
 use std::rc::Rc;
@@ -68,7 +68,9 @@ pub struct SequenceRepetition {
 /// for token sequences.
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
 pub enum KleeneOp {
+    /// Kleene star (`*`) for zero or more repetitions
     ZeroOrMore,
+    /// Kleene star (`+`) for one or more repetitions
     OneOrMore,
 }
 
@@ -83,7 +85,11 @@ pub enum TokenTree {
     /// E.g. `$var`
     MetaVar(Span, ast::Ident),
     /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
-    MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */),
+    MetaVarDecl(
+        Span,
+        ast::Ident, /* name to bind */
+        ast::Ident, /* kind of nonterminal */
+    ),
 }
 
 impl TokenTree {
@@ -131,17 +137,20 @@ impl TokenTree {
     /// Retrieve the `TokenTree`'s span.
     pub fn span(&self) -> Span {
         match *self {
-            TokenTree::Token(sp, _) |
-            TokenTree::MetaVar(sp, _) |
-            TokenTree::MetaVarDecl(sp, _, _) |
-            TokenTree::Delimited(sp, _) |
-            TokenTree::Sequence(sp, _) => sp,
+            TokenTree::Token(sp, _)
+            | TokenTree::MetaVar(sp, _)
+            | TokenTree::MetaVarDecl(sp, _, _)
+            | TokenTree::Delimited(sp, _)
+            | TokenTree::Sequence(sp, _) => sp,
         }
     }
 }
 
-pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess)
-             -> Vec<TokenTree> {
+pub fn parse(
+    input: tokenstream::TokenStream,
+    expect_matchers: bool,
+    sess: &ParseSess,
+) -> Vec<TokenTree> {
     let mut result = Vec::new();
     let mut trees = input.trees();
     while let Some(tree) = trees.next() {
@@ -154,16 +163,24 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars
                             Some(kind) => {
                                 let span = end_sp.with_lo(start_sp.lo());
                                 result.push(TokenTree::MetaVarDecl(span, ident, kind));
-                                continue
+                                continue;
                             }
                             _ => end_sp,
                         },
-                        tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
+                        tree => tree.as_ref()
+                            .map(tokenstream::TokenTree::span)
+                            .unwrap_or(span),
                     },
-                    tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
+                    tree => tree.as_ref()
+                        .map(tokenstream::TokenTree::span)
+                        .unwrap_or(start_sp),
                 };
                 sess.missing_fragment_specifiers.borrow_mut().insert(span);
-                result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident()));
+                result.push(TokenTree::MetaVarDecl(
+                    span,
+                    ident,
+                    keywords::Invalid.ident(),
+                ));
             }
             _ => result.push(tree),
         }
@@ -171,12 +188,14 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars
     result
 }
 
-fn parse_tree<I>(tree: tokenstream::TokenTree,
-                 trees: &mut I,
-                 expect_matchers: bool,
-                 sess: &ParseSess)
-                 -> TokenTree
-    where I: Iterator<Item = tokenstream::TokenTree>,
+fn parse_tree<I>(
+    tree: tokenstream::TokenTree,
+    trees: &mut I,
+    expect_matchers: bool,
+    sess: &ParseSess,
+) -> TokenTree
+where
+    I: Iterator<Item = tokenstream::TokenTree>,
 {
     match tree {
         tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
@@ -189,43 +208,69 @@ fn parse_tree<I>(tree: tokenstream::TokenTree,
                 let sequence = parse(delimited.tts.into(), expect_matchers, sess);
                 let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
                 let name_captures = macro_parser::count_names(&sequence);
-                TokenTree::Sequence(span, Rc::new(SequenceRepetition {
-                    tts: sequence,
-                    separator,
-                    op,
-                    num_captures: name_captures,
-                }))
+                TokenTree::Sequence(
+                    span,
+                    Rc::new(SequenceRepetition {
+                        tts: sequence,
+                        separator,
+                        op,
+                        num_captures: name_captures,
+                    }),
+                )
             }
             Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
                 let ident = token.ident().unwrap();
                 let span = ident_span.with_lo(span.lo());
                 if ident.name == keywords::Crate.name() {
-                    let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident };
+                    let ident = ast::Ident {
+                        name: keywords::DollarCrate.name(),
+                        ..ident
+                    };
                     TokenTree::Token(span, token::Ident(ident))
                 } else {
                     TokenTree::MetaVar(span, ident)
                 }
             }
             Some(tokenstream::TokenTree::Token(span, tok)) => {
-                let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok));
+                let msg = format!(
+                    "expected identifier, found `{}`",
+                    pprust::token_to_string(&tok)
+                );
                 sess.span_diagnostic.span_err(span, &msg);
                 TokenTree::MetaVar(span, keywords::Invalid.ident())
             }
             None => TokenTree::Token(span, token::Dollar),
         },
         tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
-        tokenstream::TokenTree::Delimited(span, delimited) => {
-            TokenTree::Delimited(span, Rc::new(Delimited {
+        tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
+            span,
+            Rc::new(Delimited {
                 delim: delimited.delim,
                 tts: parse(delimited.tts.into(), expect_matchers, sess),
-            }))
-        }
+            }),
+        ),
     }
 }
 
-fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
-                              -> (Option<token::Token>, KleeneOp)
-    where I: Iterator<Item = tokenstream::TokenTree>,
+/// Attempt to parse a single Kleene star, possibly with a separator.
+///
+/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the
+/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing
+/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator
+/// itself. Note that here we are parsing the _pattern_ itself, rather than trying to match some
+/// stream of tokens against the pattern.
+///
+/// This function will take some input iterator `input` corresponding to `span` and a parsing
+/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
+/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
+/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
+fn parse_sep_and_kleene_op<I>(
+    input: &mut I,
+    span: Span,
+    sess: &ParseSess,
+) -> (Option<token::Token>, KleeneOp)
+where
+    I: Iterator<Item = tokenstream::TokenTree>,
 {
     fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
         match *token {
@@ -235,20 +280,40 @@ fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
         }
     }
 
+    // We attempt to look at the next two token trees in `input`. I will call the first #1 and the
+    // second #2. If #1 and #2 don't match a valid KleeneOp with/without separator, that is an
+    // error, and we should emit an error on the most specific span possible.
     let span = match input.next() {
+        // #1 is a token
         Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) {
+            // #1 is a KleeneOp with no separator
             Some(op) => return (None, op),
+
+            // #1 is not a KleeneOp, but may be a separator... need to look at #2
             None => match input.next() {
+                // #2 is a token
                 Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) {
+                    // #2 is a KleeneOp, so #1 must be a separator
                     Some(op) => return (Some(tok), op),
+
+                    // #2 is not a KleeneOp... error
                     None => span,
                 },
-                tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
-            }
+
+                // #2 is not a token at all... error
+                tree => tree.as_ref()
+                    .map(tokenstream::TokenTree::span)
+                    .unwrap_or(span),
+            },
         },
-        tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
+
+        // #1 is not a token at all... error
+        tree => tree.as_ref()
+            .map(tokenstream::TokenTree::span)
+            .unwrap_or(span),
     };
 
+    // Error...
     sess.span_diagnostic.span_err(span, "expected `*` or `+`");
     (None, KleeneOp::ZeroOrMore)
 }

From 49431d49661af7a3e55743a398346903ef58f20f Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Fri, 19 Jan 2018 16:39:54 -0600
Subject: [PATCH 03/26] Add a bunch of doc comments

---
 src/libsyntax/ext/tt/quoted.rs | 75 ++++++++++++++++++++++++++++++++--
 1 file changed, 72 insertions(+), 3 deletions(-)

diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index ee87a612345..606dfcd58a2 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -18,6 +18,8 @@ use tokenstream;
 
 use std::rc::Rc;
 
+/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
+/// thatthat the delimiter itself might be `NoDelim`.
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
 pub struct Delimited {
     pub delim: token::DelimToken,
@@ -25,14 +27,17 @@ pub struct Delimited {
 }
 
 impl Delimited {
+    /// Return the opening delimiter (possibly `NoDelim`).
     pub fn open_token(&self) -> token::Token {
         token::OpenDelim(self.delim)
     }
 
+    /// Return the closing delimiter (possibly `NoDelim`).
     pub fn close_token(&self) -> token::Token {
         token::CloseDelim(self.delim)
     }
 
+    /// Return a `self::TokenTree` witha a `Span` corresponding to the opening delimiter.
     pub fn open_tt(&self, span: Span) -> TokenTree {
         let open_span = if span == DUMMY_SP {
             DUMMY_SP
@@ -42,6 +47,7 @@ impl Delimited {
         TokenTree::Token(open_span, self.open_token())
     }
 
+    /// Return a `self::TokenTree` witha a `Span` corresponding to the closing delimiter.
     pub fn close_tt(&self, span: Span) -> TokenTree {
         let close_span = if span == DUMMY_SP {
             DUMMY_SP
@@ -75,7 +81,7 @@ pub enum KleeneOp {
 }
 
 /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
-/// are "first-class" token trees.
+/// are "first-class" token trees. Useful for parsing macros.
 #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)]
 pub enum TokenTree {
     Token(Span, token::Token),
@@ -93,6 +99,7 @@ pub enum TokenTree {
 }
 
 impl TokenTree {
+    /// Return the number of tokens in the tree.
     pub fn len(&self) -> usize {
         match *self {
             TokenTree::Delimited(_, ref delimed) => match delimed.delim {
@@ -104,6 +111,8 @@ impl TokenTree {
         }
     }
 
+    /// Returns true if the given token tree contains no other tokens. This is vacuously true for
+    /// single tokens or metavar/decls, but may be false for delimited trees or sequences.
     pub fn is_empty(&self) -> bool {
         match *self {
             TokenTree::Delimited(_, ref delimed) => match delimed.delim {
@@ -115,6 +124,7 @@ impl TokenTree {
         }
     }
 
+    /// Get the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
     pub fn get_tt(&self, index: usize) -> TokenTree {
         match (self, index) {
             (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
@@ -146,15 +156,39 @@ impl TokenTree {
     }
 }
 
+/// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this
+/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
+/// collection of `TokenTree` for use in parsing a macro.
+///
+/// # Parameters
+///
+/// - `input`: a token stream to read from, the contents of which we are parsing.
+/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a
+///   macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with
+///   their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and
+///   `ident` are "matchers". They are not present in the body of a macro rule -- just in the
+///   pattern, so we pass a parameter to indicate whether to expect them or not.
+/// - `sess`: the parsing session. Any errors will be emitted to this session.
+///
+/// # Returns
+///
+/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
 pub fn parse(
     input: tokenstream::TokenStream,
     expect_matchers: bool,
     sess: &ParseSess,
 ) -> Vec<TokenTree> {
+    // Will contain the final collection of `self::TokenTree`
     let mut result = Vec::new();
+
+    // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
+    // additional trees if need be.
     let mut trees = input.trees();
     while let Some(tree) = trees.next() {
         let tree = parse_tree(tree, &mut trees, expect_matchers, sess);
+
+        // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
+        // parse out the matcher (i.e. in `$id:ident` this would parse the `:` and `ident`).
         match tree {
             TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
                 let span = match trees.next() {
@@ -182,12 +216,27 @@ pub fn parse(
                     keywords::Invalid.ident(),
                 ));
             }
+
+            // Not a metavar or no matchers allowed, so just return the tree
             _ => result.push(tree),
         }
     }
     result
 }
 
+/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a
+/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree`
+/// for use in parsing a macro.
+///
+/// Converting the given tree may involve reading more tokens.
+///
+/// # Parameters
+///
+/// - `tree`: the tree wish to convert.
+/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish
+///   converting `tree`
+/// - `expect_matchers`: same as for `parse` (see above).
+/// - `sess`: the parsing session. Any errors will be emitted to this session.
 fn parse_tree<I>(
     tree: tokenstream::TokenTree,
     trees: &mut I,
@@ -197,16 +246,24 @@ fn parse_tree<I>(
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
+    // Depending on what `tree` is, we could be parsing different parts of a macro
     match tree {
+        // `tree` is a `$` token. Look at the next token in `trees`
         tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
+            // `tree` is followed by a delimited set of token trees. This indicates the beginning
+            // of a repetition sequence in the macro (e.g. `$(pat)*`).
             Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
+                // Must have `(` not `{` or `[`
                 if delimited.delim != token::Paren {
                     let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
                     let msg = format!("expected `(`, found `{}`", tok);
                     sess.span_diagnostic.span_err(span, &msg);
                 }
+                // Parse the contents of the sequence itself
                 let sequence = parse(delimited.tts.into(), expect_matchers, sess);
+                // Get the Kleen operator and optional separator
                 let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
+                // Count the number of captured "names" (i.e. named metavars)
                 let name_captures = macro_parser::count_names(&sequence);
                 TokenTree::Sequence(
                     span,
@@ -218,6 +275,9 @@ where
                     }),
                 )
             }
+
+            // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
+            // metavariable that names the crate of the invokation.
             Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
                 let ident = token.ident().unwrap();
                 let span = ident_span.with_lo(span.lo());
@@ -231,6 +291,8 @@ where
                     TokenTree::MetaVar(span, ident)
                 }
             }
+
+            // `tree` is followed by a random token. This is an error.
             Some(tokenstream::TokenTree::Token(span, tok)) => {
                 let msg = format!(
                     "expected identifier, found `{}`",
@@ -239,9 +301,16 @@ where
                 sess.span_diagnostic.span_err(span, &msg);
                 TokenTree::MetaVar(span, keywords::Invalid.ident())
             }
+
+            // There are no more tokens. Just return the `$` we already have.
             None => TokenTree::Token(span, token::Dollar),
         },
+
+        // `tree` is an arbitrary token. Keep it.
         tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
+
+        // `tree` is the beginning of a delimited set of tokens (e.g. `(` or `{`). We need to
+        // descend into the delimited set and further parse it.
         tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
             span,
             Rc::new(Delimited {
@@ -257,8 +326,8 @@ where
 /// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the
 /// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing
 /// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator
-/// itself. Note that here we are parsing the _pattern_ itself, rather than trying to match some
-/// stream of tokens against the pattern.
+/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some
+/// stream of tokens in an invokation of a macro.
 ///
 /// This function will take some input iterator `input` corresponding to `span` and a parsing
 /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene

From ca0c0805693b08566cf118b676533be776005494 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Sun, 21 Jan 2018 16:03:47 -0600
Subject: [PATCH 04/26] Fix typos

---
 src/libsyntax/ext/tt/quoted.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 606dfcd58a2..61dc3d32f20 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -19,7 +19,7 @@ use tokenstream;
 use std::rc::Rc;
 
 /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
-/// thatthat the delimiter itself might be `NoDelim`.
+/// that the delimiter itself might be `NoDelim`.
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
 pub struct Delimited {
     pub delim: token::DelimToken,
@@ -76,7 +76,7 @@ pub struct SequenceRepetition {
 pub enum KleeneOp {
     /// Kleene star (`*`) for zero or more repetitions
     ZeroOrMore,
-    /// Kleene star (`+`) for one or more repetitions
+    /// Kleene plus (`+`) for one or more repetitions
     OneOrMore,
 }
 
@@ -261,7 +261,7 @@ where
                 }
                 // Parse the contents of the sequence itself
                 let sequence = parse(delimited.tts.into(), expect_matchers, sess);
-                // Get the Kleen operator and optional separator
+                // Get the Kleene operator and optional separator
                 let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
                 // Count the number of captured "names" (i.e. named metavars)
                 let name_captures = macro_parser::count_names(&sequence);

From 831ff775703eb5126f741fc3be1cf829ec060011 Mon Sep 17 00:00:00 2001
From: Corentin Henry <corentinhenry@gmail.com>
Date: Thu, 25 Jan 2018 15:08:48 -0800
Subject: [PATCH 05/26] implement Send for process::Command on unix

closes https://github.com/rust-lang/rust/issues/47751
---
 src/libstd/sys/unix/process/process_common.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libstd/sys/unix/process/process_common.rs b/src/libstd/sys/unix/process/process_common.rs
index c53bcdbf8e3..b09fc36dee0 100644
--- a/src/libstd/sys/unix/process/process_common.rs
+++ b/src/libstd/sys/unix/process/process_common.rs
@@ -87,6 +87,11 @@ pub enum Stdio {
     Fd(FileDesc),
 }
 
+// Command is not Send by default due to the Command.argv field containing a raw pointers. However
+// it is safe to implement Send, because anyway, these pointers point to memory owned by the
+// Command.args field.
+unsafe impl Send for Command {}
+
 impl Command {
     pub fn new(program: &OsStr) -> Command {
         let mut saw_nul = false;

From 9e6ed17c4f5befef64fa9e8a1d2b2f155c345cac Mon Sep 17 00:00:00 2001
From: Corentin Henry <corentinhenry@gmail.com>
Date: Fri, 26 Jan 2018 07:22:43 -0800
Subject: [PATCH 06/26] make Command.argv Send on unix platforms

Implementing Send for a specific field rather than the whole struct is
safer: if a field is changed/modified and becomes non-Send, we can catch
it.
---
 src/libstd/sys/unix/process/process_common.rs | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/libstd/sys/unix/process/process_common.rs b/src/libstd/sys/unix/process/process_common.rs
index b09fc36dee0..7e057401fab 100644
--- a/src/libstd/sys/unix/process/process_common.rs
+++ b/src/libstd/sys/unix/process/process_common.rs
@@ -45,7 +45,7 @@ pub struct Command {
     // other keys.
     program: CString,
     args: Vec<CString>,
-    argv: Vec<*const c_char>,
+    argv: Argv,
     env: CommandEnv<DefaultEnvKey>,
 
     cwd: Option<CString>,
@@ -58,6 +58,12 @@ pub struct Command {
     stderr: Option<Stdio>,
 }
 
+// Create a new type for argv, so that we can make it `Send`
+struct Argv(Vec<*const c_char>);
+
+// It is safe to make Argv Send, because it contains pointers to memory owned by `Command.args`
+unsafe impl Send for Argv {}
+
 // passed back to std::process with the pipes connected to the child, if any
 // were requested
 pub struct StdioPipes {
@@ -87,17 +93,12 @@ pub enum Stdio {
     Fd(FileDesc),
 }
 
-// Command is not Send by default due to the Command.argv field containing a raw pointers. However
-// it is safe to implement Send, because anyway, these pointers point to memory owned by the
-// Command.args field.
-unsafe impl Send for Command {}
-
 impl Command {
     pub fn new(program: &OsStr) -> Command {
         let mut saw_nul = false;
         let program = os2c(program, &mut saw_nul);
         Command {
-            argv: vec![program.as_ptr(), ptr::null()],
+            argv: Argv(vec![program.as_ptr(), ptr::null()]),
             program,
             args: Vec::new(),
             env: Default::default(),
@@ -116,8 +117,8 @@ impl Command {
         // Overwrite the trailing NULL pointer in `argv` and then add a new null
         // pointer.
         let arg = os2c(arg, &mut self.saw_nul);
-        self.argv[self.args.len() + 1] = arg.as_ptr();
-        self.argv.push(ptr::null());
+        self.argv.0[self.args.len() + 1] = arg.as_ptr();
+        self.argv.0.push(ptr::null());
 
         // Also make sure we keep track of the owned value to schedule a
         // destructor for this memory.
@@ -138,7 +139,7 @@ impl Command {
         self.saw_nul
     }
     pub fn get_argv(&self) -> &Vec<*const c_char> {
-        &self.argv
+        &self.argv.0
     }
 
     #[allow(dead_code)]

From 0ac465924e6ae4380b25c38cbc14f425796fa2af Mon Sep 17 00:00:00 2001
From: varkor <github@varkor.com>
Date: Fri, 26 Jan 2018 15:33:05 +0000
Subject: [PATCH 07/26] Add line numbers and columns to error messages spanning
 multiple files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If an error message is emitted that spans several files, only the
primary file currently has line and column data attached. This is
useful information, even in files other than the one in which the error
occurs. We can often work out which line and column the error
corresponds to in other files — in this case it is helpful to add them
(in the case of ambiguity, the first relevant line/column is picked,
which is still helpful than none).
---
 src/librustc_errors/emitter.rs              | 13 ++++++++++++-
 src/librustc_errors/snippet.rs              |  3 ++-
 src/test/ui/cross-file-errors/main.rs       | 16 ++++++++++++++++
 src/test/ui/cross-file-errors/main.stderr   | 11 +++++++++++
 src/test/ui/cross-file-errors/underscore.rs | 20 ++++++++++++++++++++
 src/tools/compiletest/src/runtest.rs        |  2 +-
 6 files changed, 62 insertions(+), 3 deletions(-)
 create mode 100644 src/test/ui/cross-file-errors/main.rs
 create mode 100644 src/test/ui/cross-file-errors/main.stderr
 create mode 100644 src/test/ui/cross-file-errors/underscore.rs

diff --git a/src/librustc_errors/emitter.rs b/src/librustc_errors/emitter.rs
index 58f851aea38..a9f228ca729 100644
--- a/src/librustc_errors/emitter.rs
+++ b/src/librustc_errors/emitter.rs
@@ -1013,8 +1013,19 @@ impl EmitterWriter {
 
                 // Then, the secondary file indicator
                 buffer.prepend(buffer_msg_line_offset + 1, "::: ", Style::LineNumber);
+                let loc = if let Some(first_line) = annotated_file.lines.first() {
+                    let col = if let Some(first_annotation) = first_line.annotations.first() {
+                        format!(":{}", first_annotation.start_col + 1)
+                    } else { "".to_string() };
+                    format!("{}:{}{}",
+                            annotated_file.file.name,
+                            cm.doctest_offset_line(first_line.line_index),
+                            col)
+                } else {
+                    annotated_file.file.name.to_string()
+                };
                 buffer.append(buffer_msg_line_offset + 1,
-                              &annotated_file.file.name.to_string(),
+                              &loc,
                               Style::LineAndColumn);
                 for _ in 0..max_line_num_len {
                     buffer.prepend(buffer_msg_line_offset + 1, " ", Style::NoStyle);
diff --git a/src/librustc_errors/snippet.rs b/src/librustc_errors/snippet.rs
index c2f4701999e..6035f33c822 100644
--- a/src/librustc_errors/snippet.rs
+++ b/src/librustc_errors/snippet.rs
@@ -27,7 +27,8 @@ pub struct FileInfo {
 
     /// The "primary file", if any, gets a `-->` marker instead of
     /// `>>>`, and has a line-number/column printed and not just a
-    /// filename.  It appears first in the listing. It is known to
+    /// filename (other files are not guaranteed to have line numbers
+    /// or columns). It appears first in the listing. It is known to
     /// contain at least one primary span, though primary spans (which
     /// are designated with `^^^`) may also occur in other files.
     primary_span: Option<Span>,
diff --git a/src/test/ui/cross-file-errors/main.rs b/src/test/ui/cross-file-errors/main.rs
new file mode 100644
index 00000000000..8eae79a21a9
--- /dev/null
+++ b/src/test/ui/cross-file-errors/main.rs
@@ -0,0 +1,16 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[macro_use]
+mod underscore;
+
+fn main() {
+    underscore!();
+}
diff --git a/src/test/ui/cross-file-errors/main.stderr b/src/test/ui/cross-file-errors/main.stderr
new file mode 100644
index 00000000000..a1cdae10edf
--- /dev/null
+++ b/src/test/ui/cross-file-errors/main.stderr
@@ -0,0 +1,11 @@
+error: expected expression, found `_`
+  --> $DIR/underscore.rs:18:9
+   |
+18 |         _
+   |         ^
+   | 
+  ::: $DIR/main.rs:15:5
+   |
+15 |     underscore!();
+   |     -------------- in this macro invocation
+
diff --git a/src/test/ui/cross-file-errors/underscore.rs b/src/test/ui/cross-file-errors/underscore.rs
new file mode 100644
index 00000000000..312b3b8f4dd
--- /dev/null
+++ b/src/test/ui/cross-file-errors/underscore.rs
@@ -0,0 +1,20 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// We want this file only so we can test cross-file error
+// messages, but we don't want it in an external crate.
+// ignore-test
+#![crate_type = "lib"]
+
+macro_rules! underscore {
+    () => (
+        _
+    )
+}
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index bf5fc00428d..abf62a060b8 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -1402,7 +1402,7 @@ impl<'test> TestCx<'test> {
     }
 
     /// For each `aux-build: foo/bar` annotation, we check to find the
-    /// file in a `aux` directory relative to the test itself.
+    /// file in a `auxiliary` directory relative to the test itself.
     fn compute_aux_test_paths(&self, rel_ab: &str) -> TestPaths {
         let test_ab = self.testpaths
             .file

From 077d3434aa8a2a3064afcc1c9406a49d0acf0a8d Mon Sep 17 00:00:00 2001
From: Corentin Henry <corentinhenry@gmail.com>
Date: Fri, 26 Jan 2018 07:33:58 -0800
Subject: [PATCH 08/26] add test checking that process::Command is Send

---
 src/libstd/process.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libstd/process.rs b/src/libstd/process.rs
index 5c66ac6ddde..9b2f815b713 100644
--- a/src/libstd/process.rs
+++ b/src/libstd/process.rs
@@ -1843,4 +1843,10 @@ mod tests {
         }
         assert!(events > 0);
     }
+
+    #[test]
+    fn test_command_implements_send() {
+        fn take_send_type<T: Send>(_: T) {}
+        take_send_type(Command::new(""))
+    }
 }

From aa6cc6e1898067c4311e260960bca776f3e02715 Mon Sep 17 00:00:00 2001
From: varkor <github@varkor.com>
Date: Fri, 26 Jan 2018 16:56:37 +0000
Subject: [PATCH 09/26] Fix test in macro_backtrace

---
 src/test/ui/macro_backtrace/main.stderr | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/test/ui/macro_backtrace/main.stderr b/src/test/ui/macro_backtrace/main.stderr
index 5990f71b3ca..48138ee711b 100644
--- a/src/test/ui/macro_backtrace/main.stderr
+++ b/src/test/ui/macro_backtrace/main.stderr
@@ -22,7 +22,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found
 27 |       ping!();
    |       -------- in this macro invocation
    | 
-  ::: <ping macros>
+  ::: <ping macros>:1:1
    |
 1  |   (  ) => { pong ! (  ) ; }
    |   -------------------------
@@ -42,7 +42,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found
 28 |       deep!();
    |       -------- in this macro invocation (#1)
    | 
-  ::: <deep macros>
+  ::: <deep macros>:1:1
    |
 1  |   (  ) => { foo ! (  ) ; }
    |   ------------------------
@@ -50,7 +50,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found
    |   |         in this macro invocation (#2)
    |   in this expansion of `deep!` (#1)
    | 
-  ::: <foo macros>
+  ::: <foo macros>:1:1
    |
 1  |   (  ) => { bar ! (  ) ; }
    |   ------------------------
@@ -58,7 +58,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found
    |   |         in this macro invocation (#3)
    |   in this expansion of `foo!` (#2)
    | 
-  ::: <bar macros>
+  ::: <bar macros>:1:1
    |
 1  |   (  ) => { ping ! (  ) ; }
    |   -------------------------
@@ -66,7 +66,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found
    |   |         in this macro invocation (#4)
    |   in this expansion of `bar!` (#3)
    | 
-  ::: <ping macros>
+  ::: <ping macros>:1:1
    |
 1  |   (  ) => { pong ! (  ) ; }
    |   -------------------------

From ac0c16d3b5cc5644b3311811e127411b87f3abf0 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Fri, 19 Jan 2018 19:00:29 -0600
Subject: [PATCH 10/26] Run rustfmt on /libsyntax/ext/tt/macro_parser.rs

---
 src/libsyntax/ext/tt/macro_parser.rs | 190 ++++++++++++++++-----------
 1 file changed, 113 insertions(+), 77 deletions(-)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 3e3c1618fff..a5b573f18db 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -90,8 +90,8 @@ use codemap::Spanned;
 use errors::FatalError;
 use ext::tt::quoted::{self, TokenTree};
 use parse::{Directory, ParseSess};
-use parse::parser::{PathStyle, Parser};
-use parse::token::{self, DocComment, Token, Nonterminal};
+use parse::parser::{Parser, PathStyle};
+use parse::token::{self, DocComment, Nonterminal, Token};
 use print::pprust;
 use symbol::keywords;
 use tokenstream::TokenStream;
@@ -100,7 +100,7 @@ use util::small_vector::SmallVector;
 use std::mem;
 use std::rc::Rc;
 use std::collections::HashMap;
-use std::collections::hash_map::Entry::{Vacant, Occupied};
+use std::collections::hash_map::Entry::{Occupied, Vacant};
 
 // To avoid costly uniqueness checks, we require that `MatchSeq` always has
 // a nonempty body.
@@ -182,7 +182,7 @@ fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
         match_lo: 0,
         match_cur: 0,
         match_hi: match_idx_hi,
-        sp_lo: lo
+        sp_lo: lo,
     })
 }
 
@@ -206,25 +206,27 @@ fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
 #[derive(Debug, Clone)]
 pub enum NamedMatch {
     MatchedSeq(Rc<Vec<NamedMatch>>, syntax_pos::Span),
-    MatchedNonterminal(Rc<Nonterminal>)
+    MatchedNonterminal(Rc<Nonterminal>),
 }
 
-fn nameize<I: Iterator<Item=NamedMatch>>(sess: &ParseSess, ms: &[TokenTree], mut res: I)
-                                             -> NamedParseResult {
-    fn n_rec<I: Iterator<Item=NamedMatch>>(sess: &ParseSess, m: &TokenTree, res: &mut I,
-             ret_val: &mut HashMap<Ident, Rc<NamedMatch>>)
-             -> Result<(), (syntax_pos::Span, String)> {
+fn nameize<I: Iterator<Item = NamedMatch>>(
+    sess: &ParseSess,
+    ms: &[TokenTree],
+    mut res: I,
+) -> NamedParseResult {
+    fn n_rec<I: Iterator<Item = NamedMatch>>(
+        sess: &ParseSess,
+        m: &TokenTree,
+        res: &mut I,
+        ret_val: &mut HashMap<Ident, Rc<NamedMatch>>,
+    ) -> Result<(), (syntax_pos::Span, String)> {
         match *m {
-            TokenTree::Sequence(_, ref seq) => {
-                for next_m in &seq.tts {
-                    n_rec(sess, next_m, res.by_ref(), ret_val)?
-                }
-            }
-            TokenTree::Delimited(_, ref delim) => {
-                for next_m in &delim.tts {
-                    n_rec(sess, next_m, res.by_ref(), ret_val)?;
-                }
-            }
+            TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts {
+                n_rec(sess, next_m, res.by_ref(), ret_val)?
+            },
+            TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts {
+                n_rec(sess, next_m, res.by_ref(), ret_val)?;
+            },
             TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => {
                 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
                     return Err((span, "missing fragment specifier".to_string()));
@@ -250,7 +252,7 @@ fn nameize<I: Iterator<Item=NamedMatch>>(sess: &ParseSess, ms: &[TokenTree], mut
     let mut ret_val = HashMap::new();
     for m in ms {
         match n_rec(sess, m, res.by_ref(), &mut ret_val) {
-            Ok(_) => {},
+            Ok(_) => {}
             Err((sp, msg)) => return Error(sp, msg),
         }
     }
@@ -265,18 +267,21 @@ pub enum ParseResult<T> {
     /// indicates that no rules expected the given token.
     Failure(syntax_pos::Span, Token),
     /// Fatal error (malformed macro?). Abort compilation.
-    Error(syntax_pos::Span, String)
+    Error(syntax_pos::Span, String),
 }
 
 pub fn parse_failure_msg(tok: Token) -> String {
     match tok {
         token::Eof => "unexpected end of macro invocation".to_string(),
-        _ => format!("no rules expected the token `{}`", pprust::token_to_string(&tok)),
+        _ => format!(
+            "no rules expected the token `{}`",
+            pprust::token_to_string(&tok)
+        ),
     }
 }
 
 /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
-fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
+fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) {
         id1.name == id2.name
     } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) {
@@ -290,14 +295,15 @@ fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
     (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect()
 }
 
-fn inner_parse_loop(sess: &ParseSess,
-                    cur_items: &mut SmallVector<Box<MatcherPos>>,
-                    next_items: &mut Vec<Box<MatcherPos>>,
-                    eof_items: &mut SmallVector<Box<MatcherPos>>,
-                    bb_items: &mut SmallVector<Box<MatcherPos>>,
-                    token: &Token,
-                    span: syntax_pos::Span)
-                    -> ParseResult<()> {
+fn inner_parse_loop(
+    sess: &ParseSess,
+    cur_items: &mut SmallVector<Box<MatcherPos>>,
+    next_items: &mut Vec<Box<MatcherPos>>,
+    eof_items: &mut SmallVector<Box<MatcherPos>>,
+    bb_items: &mut SmallVector<Box<MatcherPos>>,
+    token: &Token,
+    span: syntax_pos::Span,
+) -> ParseResult<()> {
     while let Some(mut item) = cur_items.pop() {
         // When unzipped trees end, remove them
         while item.idx >= item.top_elts.len() {
@@ -306,7 +312,7 @@ fn inner_parse_loop(sess: &ParseSess,
                     item.top_elts = elts;
                     item.idx = idx + 1;
                 }
-                None => break
+                None => break,
             }
         }
 
@@ -341,11 +347,16 @@ fn inner_parse_loop(sess: &ParseSess,
                 // Check if we need a separator
                 if idx == len && item.sep.is_some() {
                     // We have a separator, and it is the current token.
-                    if item.sep.as_ref().map(|sep| token_name_eq(token, sep)).unwrap_or(false) {
+                    if item.sep
+                        .as_ref()
+                        .map(|sep| token_name_eq(token, sep))
+                        .unwrap_or(false)
+                    {
                         item.idx += 1;
                         next_items.push(item);
                     }
-                } else { // we don't need a separator
+                } else {
+                    // we don't need a separator
                     item.match_cur = item.match_lo;
                     item.idx = 0;
                     cur_items.push(item);
@@ -418,12 +429,13 @@ fn inner_parse_loop(sess: &ParseSess,
     Success(())
 }
 
-pub fn parse(sess: &ParseSess,
-             tts: TokenStream,
-             ms: &[TokenTree],
-             directory: Option<Directory>,
-             recurse_into_modules: bool)
-             -> NamedParseResult {
+pub fn parse(
+    sess: &ParseSess,
+    tts: TokenStream,
+    ms: &[TokenTree],
+    directory: Option<Directory>,
+    recurse_into_modules: bool,
+) -> NamedParseResult {
     let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
     let mut cur_items = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo()));
     let mut next_items = Vec::new(); // or proceed normally
@@ -433,9 +445,16 @@ pub fn parse(sess: &ParseSess,
         let mut eof_items = SmallVector::new();
         assert!(next_items.is_empty());
 
-        match inner_parse_loop(sess, &mut cur_items, &mut next_items, &mut eof_items, &mut bb_items,
-                               &parser.token, parser.span) {
-            Success(_) => {},
+        match inner_parse_loop(
+            sess,
+            &mut cur_items,
+            &mut next_items,
+            &mut eof_items,
+            &mut bb_items,
+            &parser.token,
+            parser.span,
+        ) {
+            Success(_) => {}
             Failure(sp, tok) => return Failure(sp, tok),
             Error(sp, msg) => return Error(sp, msg),
         }
@@ -446,43 +465,56 @@ pub fn parse(sess: &ParseSess,
         /* error messages here could be improved with links to orig. rules */
         if token_name_eq(&parser.token, &token::Eof) {
             if eof_items.len() == 1 {
-                let matches = eof_items[0].matches.iter_mut().map(|dv| {
-                    Rc::make_mut(dv).pop().unwrap()
-                });
+                let matches = eof_items[0]
+                    .matches
+                    .iter_mut()
+                    .map(|dv| Rc::make_mut(dv).pop().unwrap());
                 return nameize(sess, ms, matches);
             } else if eof_items.len() > 1 {
-                return Error(parser.span, "ambiguity: multiple successful parses".to_string());
+                return Error(
+                    parser.span,
+                    "ambiguity: multiple successful parses".to_string(),
+                );
             } else {
                 return Failure(parser.span, token::Eof);
             }
         } else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 {
-            let nts = bb_items.iter().map(|item| match item.top_elts.get_tt(item.idx) {
-                TokenTree::MetaVarDecl(_, bind, name) => {
-                    format!("{} ('{}')", name, bind)
-                }
-                _ => panic!()
-            }).collect::<Vec<String>>().join(" or ");
+            let nts = bb_items
+                .iter()
+                .map(|item| match item.top_elts.get_tt(item.idx) {
+                    TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind),
+                    _ => panic!(),
+                })
+                .collect::<Vec<String>>()
+                .join(" or ");
 
-            return Error(parser.span, format!(
-                "local ambiguity: multiple parsing options: {}",
-                match next_items.len() {
-                    0 => format!("built-in NTs {}.", nts),
-                    1 => format!("built-in NTs {} or 1 other option.", nts),
-                    n => format!("built-in NTs {} or {} other options.", nts, n),
-                }
-            ));
+            return Error(
+                parser.span,
+                format!(
+                    "local ambiguity: multiple parsing options: {}",
+                    match next_items.len() {
+                        0 => format!("built-in NTs {}.", nts),
+                        1 => format!("built-in NTs {} or 1 other option.", nts),
+                        n => format!("built-in NTs {} or {} other options.", nts, n),
+                    }
+                ),
+            );
         } else if bb_items.is_empty() && next_items.is_empty() {
             return Failure(parser.span, parser.token);
         } else if !next_items.is_empty() {
             /* Now process the next token */
             cur_items.extend(next_items.drain(..));
             parser.bump();
-        } else /* bb_items.len() == 1 */ {
+        } else
+        /* bb_items.len() == 1 */
+        {
             let mut item = bb_items.pop().unwrap();
             if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) {
                 let match_cur = item.match_cur;
-                item.push_match(match_cur,
-                    MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.name.as_str()))));
+                item.push_match(
+                    match_cur,
+                    MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.name.as_str()))),
+                );
                 item.idx += 1;
                 item.match_cur += 1;
             } else {
@@ -512,20 +544,21 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
         "expr" => token.can_begin_expr(),
         "ty" => token.can_begin_type(),
         "ident" => token.is_ident(),
-        "vis" => match *token { // The follow-set of :vis + "priv" keyword + interpolated
+        "vis" => match *token {
+            // The follow-set of :vis + "priv" keyword + interpolated
             Token::Comma | Token::Ident(_) | Token::Interpolated(_) => true,
             _ => token.can_begin_type(),
         },
         "block" => match *token {
             Token::OpenDelim(token::Brace) => true,
             Token::Interpolated(ref nt) => match nt.0 {
-                token::NtItem(_) |
-                token::NtPat(_) |
-                token::NtTy(_) |
-                token::NtIdent(_) |
-                token::NtMeta(_) |
-                token::NtPath(_) |
-                token::NtVis(_) => false, // none of these may start with '{'.
+                token::NtItem(_)
+                | token::NtPat(_)
+                | token::NtTy(_)
+                | token::NtIdent(_)
+                | token::NtMeta(_)
+                | token::NtPath(_)
+                | token::NtVis(_) => false, // none of these may start with '{'.
                 _ => true,
             },
             _ => false,
@@ -591,12 +624,15 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
         "ident" => match p.token {
             token::Ident(sn) => {
                 p.bump();
-                token::NtIdent(Spanned::<Ident>{node: sn, span: p.prev_span})
+                token::NtIdent(Spanned::<Ident> {
+                    node: sn,
+                    span: p.prev_span,
+                })
             }
             _ => {
                 let token_str = pprust::token_to_string(&p.token);
-                p.fatal(&format!("expected ident, found {}",
-                                 &token_str[..])).emit();
+                p.fatal(&format!("expected ident, found {}", &token_str[..]))
+                    .emit();
                 FatalError.raise()
             }
         },
@@ -606,6 +642,6 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
         "lifetime" => token::NtLifetime(p.expect_lifetime()),
         // this is not supposed to happen, since it has been checked
         // when compiling the macro.
-        _ => p.span_bug(sp, "invalid fragment specifier")
+        _ => p.span_bug(sp, "invalid fragment specifier"),
     }
 }

From 0d7f193dd358cdc13506cac2e0b84fc473b628be Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Fri, 19 Jan 2018 20:47:39 -0600
Subject: [PATCH 11/26] Added a bunch of comments to macro_parser.rs

---
 src/libsyntax/ext/tt/macro_parser.rs | 72 ++++++++++++++++++++++++----
 1 file changed, 62 insertions(+), 10 deletions(-)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index a5b573f18db..cb671d75a00 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -429,6 +429,17 @@ fn inner_parse_loop(
     Success(())
 }
 
+/// Parse the given set of token trees (`ms`), possibly consuming additional token trees from the
+/// tokenstream (`tts`).
+///
+/// # Parameters
+///
+/// - `sess`: The session into which errors are emitted
+/// - `tts`: The tokenstream from which additional token trees may be consumed if needed
+/// - `ms`: The token trees we want to parse as macros
+/// - `directory`: Information about the file locations (needed for the black-box parser)
+/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box
+///   parser)
 pub fn parse(
     sess: &ParseSess,
     tts: TokenStream,
@@ -436,15 +447,28 @@ pub fn parse(
     directory: Option<Directory>,
     recurse_into_modules: bool,
 ) -> NamedParseResult {
+    // Create a parser that can be used for the "black box" parts.
     let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
+
+    // A queue of possible matcher positions. We initialize it with the matcher position in which
+    // the "dot" is before the first token of the first token tree. `inner_parse_loop` then
+    // processes all of these possible matcher positions and produces posible next positions into
+    // `next_items`. After some post-processing, the contents of `next_items` replenish
+    // `cur_items` and we start over again.
     let mut cur_items = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo()));
-    let mut next_items = Vec::new(); // or proceed normally
+    let mut next_items = Vec::new();
 
     loop {
-        let mut bb_items = SmallVector::new(); // black-box parsed by parser.rs
+        // Matcher positions black-box parsed by parser.rs (`parser`)
+        let mut bb_items = SmallVector::new();
+
+        // Matcher positions that would be valid if the macro invocation was over now
         let mut eof_items = SmallVector::new();
         assert!(next_items.is_empty());
 
+        // Process `cur_items` until either we have finished the input or we need to get some
+        // parsing from the black-box parser done. The result is that `next_items` will contain a
+        // bunch of possible next matcher positions in `next_items`.
         match inner_parse_loop(
             sess,
             &mut cur_items,
@@ -462,7 +486,12 @@ pub fn parse(
         // inner parse loop handled all cur_items, so it's empty
         assert!(cur_items.is_empty());
 
-        /* error messages here could be improved with links to orig. rules */
+        // We need to do some post processing after the `inner_parser_loop`.
+        //
+        // Error messages here could be improved with links to original rules.
+
+        // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
+        // either the parse is ambiguous (which should never happen) or their is a syntax error.
         if token_name_eq(&parser.token, &token::Eof) {
             if eof_items.len() == 1 {
                 let matches = eof_items[0]
@@ -478,7 +507,10 @@ pub fn parse(
             } else {
                 return Failure(parser.span, token::Eof);
             }
-        } else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 {
+        }
+        // Another possibility is that we need to call out to parse some rust nonterminal
+        // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong.
+        else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 {
             let nts = bb_items
                 .iter()
                 .map(|item| match item.top_elts.get_tt(item.idx) {
@@ -499,15 +531,23 @@ pub fn parse(
                     }
                 ),
             );
-        } else if bb_items.is_empty() && next_items.is_empty() {
+        }
+        // If there are no posible next positions AND we aren't waiting for the black-box parser,
+        // then their is a syntax error.
+        else if bb_items.is_empty() && next_items.is_empty() {
             return Failure(parser.span, parser.token);
-        } else if !next_items.is_empty() {
-            /* Now process the next token */
+        }
+        // Dump all possible `next_items` into `cur_items` for the next iteration.
+        else if !next_items.is_empty() {
+            // Now process the next token
             cur_items.extend(next_items.drain(..));
             parser.bump();
-        } else
-        /* bb_items.len() == 1 */
-        {
+        }
+        // Finally, we have the case where we need to call the black-box parser to get some
+        // nonterminal.
+        else {
+            assert_eq!(bb_items.len(), 1);
+
             let mut item = bb_items.pop().unwrap();
             if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) {
                 let match_cur = item.match_cur;
@@ -595,6 +635,18 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
     }
 }
 
+/// A call to the "black-box" parser to parse some rust nonterminal.
+///
+/// # Parameters
+///
+/// - `p`: the "black-box" parser to use
+/// - `sp`: the `Span` we want to parse
+/// - `name`: the name of the metavar _matcher_ we want to match (e.g. `tt`, `ident`, `block`,
+///   etc...)
+///
+/// # Returns
+///
+/// The parsed nonterminal.
 fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
     if name == "tt" {
         return token::NtTT(p.parse_token_tree());

From 6d4ed65585ebdd1e5cd7aa37936682675825b919 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Wed, 24 Jan 2018 22:03:57 -0600
Subject: [PATCH 12/26] Added lots of comments + minor reorganization

---
 src/libsyntax/ext/tt/macro_parser.rs | 134 +++++++++++++++++++--------
 1 file changed, 97 insertions(+), 37 deletions(-)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index cb671d75a00..28d4f5f832f 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -102,9 +102,10 @@ use std::rc::Rc;
 use std::collections::HashMap;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
 
-// To avoid costly uniqueness checks, we require that `MatchSeq` always has
-// a nonempty body.
+// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body.
 
+/// Either a sequence of token trees or a single one. This is used as the representation of the
+/// sequence of tokens that make up a matcher.
 #[derive(Clone)]
 enum TokenTreeOrTokenTreeVec {
     Tt(TokenTree),
@@ -112,6 +113,7 @@ enum TokenTreeOrTokenTreeVec {
 }
 
 impl TokenTreeOrTokenTreeVec {
+    /// Returns the number of constituent token trees of `self`.
     fn len(&self) -> usize {
         match *self {
             TtSeq(ref v) => v.len(),
@@ -119,6 +121,7 @@ impl TokenTreeOrTokenTreeVec {
         }
     }
 
+    /// The the `index`-th token tree of `self`.
     fn get_tt(&self, index: usize) -> TokenTree {
         match *self {
             TtSeq(ref v) => v[index].clone(),
@@ -127,36 +130,90 @@ impl TokenTreeOrTokenTreeVec {
     }
 }
 
-/// an unzipping of `TokenTree`s
+/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`.
+///
+/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have
+/// descended into.
 #[derive(Clone)]
 struct MatcherTtFrame {
+    /// The "parent" matcher that we are descending into.
     elts: TokenTreeOrTokenTreeVec,
+    /// The position of the "dot" in `elts` at the time we descended.
     idx: usize,
 }
 
+/// Represents a single "position" (aka "matcher position", aka "item"), as described in the module
+/// documentation.
 #[derive(Clone)]
 struct MatcherPos {
-    stack: Vec<MatcherTtFrame>,
+    /// The token or sequence of tokens that make up the matcher
     top_elts: TokenTreeOrTokenTreeVec,
-    sep: Option<Token>,
+    /// The position of the "dot" in this matcher
     idx: usize,
-    up: Option<Box<MatcherPos>>,
-    matches: Vec<Rc<Vec<NamedMatch>>>,
-    match_lo: usize,
-    match_cur: usize,
-    match_hi: usize,
+    /// The beginning position in the source that the beginning of this matcher corresponds to. In
+    /// other words, the token in the source at `sp_lo` is matched against the first token of the
+    /// matcher.
     sp_lo: BytePos,
+
+    /// For each named metavar in the matcher, we keep track of token trees matched against the
+    /// metavar by the black box parser. In particular, there may be more than one match per
+    /// metavar if we are in a repetition (each repetition matches each of the variables).
+    /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the
+    /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of
+    /// the current position of the `self` matcher position in the shared `matches` list.
+    matches: Vec<Rc<Vec<NamedMatch>>>,
+    /// The position in `matches` corresponding to the first metavar in this matcher's sequence of
+    /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
+    /// to `matches[match_lo]`.
+    match_lo: usize,
+    /// The position in `matches` corresponding to the metavar we are currently trying to match
+    /// against the source token stream. `match_lo <= match_cur <= match_hi`.
+    match_cur: usize,
+    /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar
+    /// in this matcher.
+    match_hi: usize,
+
+    // Specifically used if we are matching a repetition. If we aren't both should be `None`.
+    /// The separator if we are in a repetition
+    sep: Option<Token>,
+    /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
+    /// before we enter the sequence.
+    up: Option<Box<MatcherPos>>,
+
+    // Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from
+    // a delimited token tree (e.g. something wrapped in `(` `)`) or to get the contents of a doc
+    // comment...
+    /// When matching against matchers with nested delimited submatchers (e.g. `pat ( pat ( .. )
+    /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
+    /// that where the bottom of the stack is the outermost matcher.
+    // Also, throughout the comments, this "descent" is often referred to as "unzipping"...
+    stack: Vec<MatcherTtFrame>,
 }
 
 impl MatcherPos {
+    /// Add `m` as a named match for the `idx`-th metavar.
     fn push_match(&mut self, idx: usize, m: NamedMatch) {
         let matches = Rc::make_mut(&mut self.matches[idx]);
         matches.push(m);
     }
 }
 
+/// Represents the possible results of an attempted parse.
+pub enum ParseResult<T> {
+    /// Parsed successfully.
+    Success(T),
+    /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
+    /// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
+    Failure(syntax_pos::Span, Token),
+    /// Fatal error (malformed macro?). Abort compilation.
+    Error(syntax_pos::Span, String),
+}
+
+/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es.
+/// This represents the mapping of metavars to the token trees they bind to.
 pub type NamedParseResult = ParseResult<HashMap<Ident, Rc<NamedMatch>>>;
 
+/// Count how many metavars are named in the given matcher `ms`.
 pub fn count_names(ms: &[TokenTree]) -> usize {
     ms.iter().fold(0, |count, elt| {
         count + match *elt {
@@ -169,20 +226,38 @@ pub fn count_names(ms: &[TokenTree]) -> usize {
     })
 }
 
+/// Initialize `len` empty shared `Vec`s to be used to store matches of metavars.
+fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
+    (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect()
+}
+
+/// Generate the top-level matcher position in which the "dot" is before the first token of the
+/// matcher `ms` and we are going to start matching at position `lo` in the source.
 fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
     let match_idx_hi = count_names(&ms[..]);
     let matches = create_matches(match_idx_hi);
     Box::new(MatcherPos {
-        stack: vec![],
-        top_elts: TtSeq(ms),
-        sep: None,
+        // Start with the top level matcher given to us
+        top_elts: TtSeq(ms), // "elts" is an abbr. for "elements"
+        // The "dot" is before the first token of the matcher
         idx: 0,
-        up: None,
+        // We start matching with byte `lo` in the source code
+        sp_lo: lo,
+
+        // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`.
+        // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since
+        // we haven't actually matched anything yet.
         matches,
         match_lo: 0,
         match_cur: 0,
         match_hi: match_idx_hi,
-        sp_lo: lo,
+
+        // Haven't descended into any delimiters, so empty stack
+        stack: vec![],
+
+        // Haven't descended into any sequences, so both of these are `None`
+        sep: None,
+        up: None,
     })
 }
 
@@ -202,7 +277,6 @@ fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
 /// token tree. The depth of the `NamedMatch` structure will therefore depend
 /// only on the nesting depth of `ast::TTSeq`s in the originating
 /// token tree it was derived from.
-
 #[derive(Debug, Clone)]
 pub enum NamedMatch {
     MatchedSeq(Rc<Vec<NamedMatch>>, syntax_pos::Span),
@@ -260,16 +334,6 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
     Success(ret_val)
 }
 
-pub enum ParseResult<T> {
-    Success(T),
-    /// Arm failed to match. If the second parameter is `token::Eof`, it
-    /// indicates an unexpected end of macro invocation. Otherwise, it
-    /// indicates that no rules expected the given token.
-    Failure(syntax_pos::Span, Token),
-    /// Fatal error (malformed macro?). Abort compilation.
-    Error(syntax_pos::Span, String),
-}
-
 pub fn parse_failure_msg(tok: Token) -> String {
     match tok {
         token::Eof => "unexpected end of macro invocation".to_string(),
@@ -291,10 +355,6 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     }
 }
 
-fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
-    (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect()
-}
-
 fn inner_parse_loop(
     sess: &ParseSess,
     cur_items: &mut SmallVector<Box<MatcherPos>>,
@@ -429,14 +489,14 @@ fn inner_parse_loop(
     Success(())
 }
 
-/// Parse the given set of token trees (`ms`), possibly consuming additional token trees from the
-/// tokenstream (`tts`).
+/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts`
+/// against it and return the match.
 ///
 /// # Parameters
 ///
 /// - `sess`: The session into which errors are emitted
-/// - `tts`: The tokenstream from which additional token trees may be consumed if needed
-/// - `ms`: The token trees we want to parse as macros
+/// - `tts`: The tokenstream we are matching against the pattern `ms`
+/// - `ms`: A sequence of token trees representing a pattern against which we are matching
 /// - `directory`: Information about the file locations (needed for the black-box parser)
 /// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box
 ///   parser)
@@ -451,10 +511,10 @@ pub fn parse(
     let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
 
     // A queue of possible matcher positions. We initialize it with the matcher position in which
-    // the "dot" is before the first token of the first token tree. `inner_parse_loop` then
+    // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then
     // processes all of these possible matcher positions and produces posible next positions into
-    // `next_items`. After some post-processing, the contents of `next_items` replenish
-    // `cur_items` and we start over again.
+    // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items`
+    // and we start over again.
     let mut cur_items = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo()));
     let mut next_items = Vec::new();
 

From b01b481db3ed9e13ab2fedc711f7cdabc8c1c53c Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Wed, 24 Jan 2018 22:59:11 -0600
Subject: [PATCH 13/26] Added/improved comments

---
 src/libsyntax/ext/tt/macro_parser.rs | 78 ++++++++++++++++++++++------
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 28d4f5f832f..f6c4359167f 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -161,6 +161,12 @@ struct MatcherPos {
     /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the
     /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of
     /// the current position of the `self` matcher position in the shared `matches` list.
+    ///
+    /// Also, note that while we are descending into a sequence, matchers are given their own
+    /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add
+    /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
+    /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
+    /// wants the shared `matches`, one should use `up.matches`.
     matches: Vec<Rc<Vec<NamedMatch>>>,
     /// The position in `matches` corresponding to the first metavar in this matcher's sequence of
     /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
@@ -255,7 +261,7 @@ fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
         // Haven't descended into any delimiters, so empty stack
         stack: vec![],
 
-        // Haven't descended into any sequences, so both of these are `None`
+        // Haven't descended into any sequences, so both of these are `None`.
         sep: None,
         up: None,
     })
@@ -355,6 +361,28 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     }
 }
 
+/// Process the matcher positions of `cur_items` until it is empty. In the process, this will
+/// produce more items in `next_items`, `eof_items`, and `bb_items`.
+///
+/// For more info about the how this happens, see the module-level doc comments and the inline
+/// comments of this function.
+///
+/// # Parameters
+///
+/// - `sess`: the parsing session into which errors are emitted.
+/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a
+///   successful execution of this function.
+/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in
+///   the function `parse`.
+/// - `eof_items`: the set of items that would be valid if this was the EOF.
+/// - `bb_items`: the set of items that are waiting for the black-box parser.
+/// - `token`: the current token of the parser.
+/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match
+///   against the matcher positions in `cur_items`.
+///
+/// # Returns
+///
+/// A `ParseResult`. Note that matches are kept track of through the items generated.
 fn inner_parse_loop(
     sess: &ParseSess,
     cur_items: &mut SmallVector<Box<MatcherPos>>,
@@ -364,8 +392,11 @@ fn inner_parse_loop(
     token: &Token,
     span: syntax_pos::Span,
 ) -> ParseResult<()> {
+    // Pop items from `cur_items` until it is empty.
     while let Some(mut item) = cur_items.pop() {
-        // When unzipped trees end, remove them
+        // When unzipped trees end, remove them. This corresponds to backtracking out of a
+        // delimited submatcher into which we already descended. In backtracking out again, we need
+        // to advance the "dot" past the delimiters in the outer matcher.
         while item.idx >= item.top_elts.len() {
             match item.stack.pop() {
                 Some(MatcherTtFrame { elts, idx }) => {
@@ -376,37 +407,46 @@ fn inner_parse_loop(
             }
         }
 
+        // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in
+        // the matcher (`len`).
         let idx = item.idx;
         let len = item.top_elts.len();
 
-        // at end of sequence
+        // If `idx >= len`, then we are at or past the end of the matcher of `item`.
         if idx >= len {
-            // We are repeating iff there is a parent
+            // We are repeating iff there is a parent. If the matcher is inside of a repetition,
+            // then we could be at the end of a sequence or at the beginning of the next
+            // repetition.
             if item.up.is_some() {
-                // Disregarding the separator, add the "up" case to the tokens that should be
-                // examined.
-                // (remove this condition to make trailing seps ok)
+                // At this point, regardless of whether there is a separator, we should add all
+                // matches from the complete repetition of the sequence to the shared, top-level
+                // `matches` list (actually, `up.matches`, which could itself not be the top-level,
+                // but anyway...). Moreover, we add another item to `cur_items` in which the "dot"
+                // is at the end of the `up` matcher. This ensures that the "dot" in the `up`
+                // matcher is also advanced sufficiently.
+                //
+                // NOTE: removing the condition `idx == len` allows trailing separators.
                 if idx == len {
+                    // Get the `up` matcher
                     let mut new_pos = item.up.clone().unwrap();
 
-                    // update matches (the MBE "parse tree") by appending
-                    // each tree as a subtree.
-
-                    // Only touch the binders we have actually bound
+                    // Add matches from this repetition to the `matches` of `up`
                     for idx in item.match_lo..item.match_hi {
                         let sub = item.matches[idx].clone();
                         let span = span.with_lo(item.sp_lo);
                         new_pos.push_match(idx, MatchedSeq(sub, span));
                     }
 
+                    // Move the "dot" past the repetition in `up`
                     new_pos.match_cur = item.match_hi;
                     new_pos.idx += 1;
                     cur_items.push(new_pos);
                 }
 
-                // Check if we need a separator
+                // Check if we need a separator.
                 if idx == len && item.sep.is_some() {
-                    // We have a separator, and it is the current token.
+                    // We have a separator, and it is the current token. We can advance past the
+                    // separator token.
                     if item.sep
                         .as_ref()
                         .map(|sep| token_name_eq(token, sep))
@@ -415,14 +455,18 @@ fn inner_parse_loop(
                         item.idx += 1;
                         next_items.push(item);
                     }
-                } else {
-                    // we don't need a separator
+                }
+                // We don't need a separator. Move the "dot" back to the beginning of the matcher
+                // and try to match again.
+                else {
                     item.match_cur = item.match_lo;
                     item.idx = 0;
                     cur_items.push(item);
                 }
-            } else {
-                // We aren't repeating, so we must be potentially at the end of the input.
+            }
+            // If we are not in a repetition, then being at the end of a matcher means that we have
+            // reached the potential end of the input.
+            else {
                 eof_items.push(item);
             }
         } else {

From 02d1d92878181a1c99b6d6029f44fbbb91bff499 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Wed, 24 Jan 2018 23:10:39 -0600
Subject: [PATCH 14/26] Still more comments

---
 src/libsyntax/ext/tt/macro_parser.rs | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index f6c4359167f..e15198cc383 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -469,9 +469,13 @@ fn inner_parse_loop(
             else {
                 eof_items.push(item);
             }
-        } else {
+        }
+        // We are in the middle of a matcher.
+        else {
+            // Look at what token in the matcher we are trying to match the current token (`token`)
+            // against. Depending on that, we may generate new items.
             match item.top_elts.get_tt(idx) {
-                /* need to descend into sequence */
+                // Need to descend into a sequence
                 TokenTree::Sequence(sp, seq) => {
                     if seq.op == quoted::KleeneOp::ZeroOrMore {
                         // Examine the case where there are 0 matches of this sequence
@@ -499,11 +503,16 @@ fn inner_parse_loop(
                         top_elts: Tt(TokenTree::Sequence(sp, seq)),
                     }));
                 }
+
+                // We need to match a metavar (but the identifier is invalid)... this is an error
                 TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => {
                     if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
                         return Error(span, "missing fragment specifier".to_string());
                     }
                 }
+
+                // We need to match a metavar with a valid ident... call out to the black-box
+                // parser by adding an item to `bb_items`.
                 TokenTree::MetaVarDecl(_, _, id) => {
                     // Built-in nonterminals never start with these tokens,
                     // so we can eliminate them from consideration.
@@ -511,6 +520,13 @@ fn inner_parse_loop(
                         bb_items.push(item);
                     }
                 }
+
+                // We need to descend into a delimited submatcher or a doc comment. To do this, we
+                // push the current matcher onto a stack and push a new item containing the
+                // submatcher onto `cur_items`.
+                //
+                // At the beginning of the loop, if we reach the end of the delimited submatcher,
+                // we pop the stack to backtrack out of the descent.
                 seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, DocComment(..)) => {
                     let lower_elts = mem::replace(&mut item.top_elts, Tt(seq));
                     let idx = item.idx;
@@ -521,15 +537,23 @@ fn inner_parse_loop(
                     item.idx = 0;
                     cur_items.push(item);
                 }
+
+                // We just matched a normal token. We can just advance the parser.
                 TokenTree::Token(_, ref t) if token_name_eq(t, token) => {
                     item.idx += 1;
                     next_items.push(item);
                 }
+
+                // There was another token that was not `token`... This means we can't add any
+                // rules. NOTE that this is not necessarily an error unless _all_ items in
+                // `cur_items` end up doing this. There may still be some other matchers that do
+                // end up working out.
                 TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
             }
         }
     }
 
+    // Yay a successful parse (so far)!
     Success(())
 }
 

From e2d558ad56d03bdc59ab2aaa28aaedf9172a0539 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Wed, 24 Jan 2018 23:23:01 -0600
Subject: [PATCH 15/26] A few more comments

---
 src/libsyntax/ext/tt/macro_parser.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index e15198cc383..af18801c97e 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -289,11 +289,17 @@ pub enum NamedMatch {
     MatchedNonterminal(Rc<Nonterminal>),
 }
 
+/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
+/// and an iterator of items that matched input and produces a `NamedParseResult`.
 fn nameize<I: Iterator<Item = NamedMatch>>(
     sess: &ParseSess,
     ms: &[TokenTree],
     mut res: I,
 ) -> NamedParseResult {
+    // Recursively descend into each type of matcher (e.g. sequences, delimited, metavars) and make
+    // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one
+    // binding, then there is an error. If it does, then we insert the binding into the
+    // `NamedParseResult`.
     fn n_rec<I: Iterator<Item = NamedMatch>>(
         sess: &ParseSess,
         m: &TokenTree,
@@ -340,6 +346,8 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
     Success(ret_val)
 }
 
+/// Generate an appropriate parsing failure message. For EOF, this is "unexpected end...". For
+/// other tokens, this is "unexpected token...".
 pub fn parse_failure_msg(tok: Token) -> String {
     match tok {
         token::Eof => "unexpected end of macro invocation".to_string(),

From a21b7b3b162932011b83f4703fb87030c216e962 Mon Sep 17 00:00:00 2001
From: varkor <github@varkor.com>
Date: Sat, 27 Jan 2018 15:16:42 +0000
Subject: [PATCH 16/26] Improve formatting of else block

---
 src/librustc_errors/emitter.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/librustc_errors/emitter.rs b/src/librustc_errors/emitter.rs
index a9f228ca729..63c4ae5561c 100644
--- a/src/librustc_errors/emitter.rs
+++ b/src/librustc_errors/emitter.rs
@@ -1016,7 +1016,9 @@ impl EmitterWriter {
                 let loc = if let Some(first_line) = annotated_file.lines.first() {
                     let col = if let Some(first_annotation) = first_line.annotations.first() {
                         format!(":{}", first_annotation.start_col + 1)
-                    } else { "".to_string() };
+                    } else {
+                        "".to_string()
+                    };
                     format!("{}:{}{}",
                             annotated_file.file.name,
                             cm.doctest_offset_line(first_line.line_index),

From 2497d10ff34bf4d03283e4562143762426492789 Mon Sep 17 00:00:00 2001
From: gnzlbg <gnzlbg@users.noreply.github.com>
Date: Sun, 28 Jan 2018 13:47:06 +0100
Subject: [PATCH 17/26] Whitelist aes x86 feature flag

Required to fix https://github.com/rust-lang-nursery/stdsimd/issues/295 in stdsimd.

r? @alexcrichton
---
 src/librustc_trans/llvm_util.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/librustc_trans/llvm_util.rs b/src/librustc_trans/llvm_util.rs
index 15988008de2..39ef3db9dc2 100644
--- a/src/librustc_trans/llvm_util.rs
+++ b/src/librustc_trans/llvm_util.rs
@@ -88,7 +88,7 @@ const X86_WHITELIST: &'static [&'static str] = &["avx\0", "avx2\0", "bmi\0", "bm
                                                  "ssse3\0", "tbm\0", "lzcnt\0", "popcnt\0",
                                                  "sse4a\0", "rdrnd\0", "rdseed\0", "fma\0",
                                                  "xsave\0", "xsaveopt\0", "xsavec\0",
-                                                 "xsaves\0",
+                                                 "xsaves\0", "aes\0",
                                                  "avx512bw\0", "avx512cd\0",
                                                  "avx512dq\0", "avx512er\0",
                                                  "avx512f\0", "avx512ifma\0",

From b32dbbc67e4ffa15c3abed2b6329f5029da7671b Mon Sep 17 00:00:00 2001
From: gnzlbg <gnzlbg@users.noreply.github.com>
Date: Sun, 28 Jan 2018 18:50:03 +0100
Subject: [PATCH 18/26] Whitelist v7 feature for ARM and AARCH64.

Needed for `v7` features in `coresimd`.

See https://github.com/rust-lang-nursery/stdsimd/blob/b2f7be24d5043a88427f9a5258ca9a51ede6d029/coresimd/src/arm/v7.rs#L40 which used to work but doesn't anymore.

r? alexcrichton
---
 src/librustc_trans/llvm_util.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/librustc_trans/llvm_util.rs b/src/librustc_trans/llvm_util.rs
index 15988008de2..a517c29032d 100644
--- a/src/librustc_trans/llvm_util.rs
+++ b/src/librustc_trans/llvm_util.rs
@@ -79,9 +79,9 @@ unsafe fn configure_llvm(sess: &Session) {
 // detection code will walk past the end of the feature array,
 // leading to crashes.
 
-const ARM_WHITELIST: &'static [&'static str] = &["neon\0", "vfp2\0", "vfp3\0", "vfp4\0"];
+const ARM_WHITELIST: &'static [&'static str] = &["neon\0", "v7\0", "vfp2\0", "vfp3\0", "vfp4\0"];
 
-const AARCH64_WHITELIST: &'static [&'static str] = &["neon\0"];
+const AARCH64_WHITELIST: &'static [&'static str] = &["neon\0", "v7\0"];
 
 const X86_WHITELIST: &'static [&'static str] = &["avx\0", "avx2\0", "bmi\0", "bmi2\0", "sse\0",
                                                  "sse2\0", "sse3\0", "sse4.1\0", "sse4.2\0",

From 7b4cbbd12d88c8e64d9c7aa1e326c7c78f2a7ed9 Mon Sep 17 00:00:00 2001
From: Corey Farwell <coreyf@rwell.org>
Date: Sun, 28 Jan 2018 21:50:01 -0500
Subject: [PATCH 19/26] Document that `Index` ops can panic on `HashMap` &
 `BTreeMap`.

Fixes https://github.com/rust-lang/rust/issues/47011.
---
 src/liballoc/btree/map.rs          | 5 +++++
 src/libstd/collections/hash/map.rs | 9 +++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/liballoc/btree/map.rs b/src/liballoc/btree/map.rs
index b114dc640fb..b320bed5432 100644
--- a/src/liballoc/btree/map.rs
+++ b/src/liballoc/btree/map.rs
@@ -1748,6 +1748,11 @@ impl<'a, K: Ord, Q: ?Sized, V> Index<&'a Q> for BTreeMap<K, V>
 {
     type Output = V;
 
+    /// Returns a reference to the value corresponding to the supplied key.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the key is not present in the `BTreeMap`.
     #[inline]
     fn index(&self, key: &Q) -> &V {
         self.get(key).expect("no entry found for key")
diff --git a/src/libstd/collections/hash/map.rs b/src/libstd/collections/hash/map.rs
index b01420f36a0..82a687ae5e4 100644
--- a/src/libstd/collections/hash/map.rs
+++ b/src/libstd/collections/hash/map.rs
@@ -1384,9 +1384,14 @@ impl<'a, K, Q: ?Sized, V, S> Index<&'a Q> for HashMap<K, V, S>
 {
     type Output = V;
 
+    /// Returns a reference to the value corresponding to the supplied key.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the key is not present in the `HashMap`.
     #[inline]
-    fn index(&self, index: &Q) -> &V {
-        self.get(index).expect("no entry found for key")
+    fn index(&self, key: &Q) -> &V {
+        self.get(key).expect("no entry found for key")
     }
 }
 

From e09a8bd70c4804234af900f19fbb883045ba8d0d Mon Sep 17 00:00:00 2001
From: Mark Simulacrum <mark.simulacrum@gmail.com>
Date: Sun, 28 Jan 2018 17:09:47 -0700
Subject: [PATCH 20/26] Add per-stage RUSTFLAGS: RUSTFLAGS_STAGE_{0,1,2} and
 RUSTFLAGS_STAGE_NOT_0

Fixes #47658.
---
 src/bootstrap/builder.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 79058984b13..1272643edd2 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -469,6 +469,18 @@ impl<'a> Builder<'a> {
             stage = compiler.stage;
         }
 
+        let mut extra_args = env::var(&format!("RUSTFLAGS_STAGE_{}", stage)).unwrap_or_default();
+        if stage != 0 {
+            let s = env::var("RUSTFLAGS_STAGE_NOT_0").unwrap_or_default();
+            extra_args.push_str(" ");
+            extra_args.push_str(&s);
+        }
+
+        if !extra_args.is_empty() {
+            cargo.env("RUSTFLAGS",
+                format!("{} {}", env::var("RUSTFLAGS").unwrap_or_default(), extra_args));
+        }
+
         // Customize the compiler we're running. Specify the compiler to cargo
         // as our shim and then pass it some various options used to configure
         // how the actual compiler itself is called.

From 8389b66c1883a44d1d79b932313a40d9c6eb116e Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Mon, 29 Jan 2018 17:11:09 +0100
Subject: [PATCH 21/26] Increase test coverage of use_nested_groups

---
 src/test/run-pass/use-nested-groups.rs     |  7 ++++++
 src/test/ui/use-nested-groups-error.rs     | 27 ++++++++++++++++++++++
 src/test/ui/use-nested-groups-error.stderr |  8 +++++++
 3 files changed, 42 insertions(+)
 create mode 100644 src/test/ui/use-nested-groups-error.rs
 create mode 100644 src/test/ui/use-nested-groups-error.stderr

diff --git a/src/test/run-pass/use-nested-groups.rs b/src/test/run-pass/use-nested-groups.rs
index 74a82afd462..a28f8da9ff8 100644
--- a/src/test/run-pass/use-nested-groups.rs
+++ b/src/test/run-pass/use-nested-groups.rs
@@ -24,12 +24,19 @@ mod a {
     }
 }
 
+// Test every possible part of the syntax
 use a::{B, d::{self, *, g::H}};
 
+// Test a more common use case
+use std::sync::{Arc, atomic::{AtomicBool, Ordering}};
+
 fn main() {
     let _: B;
     let _: E;
     let _: F;
     let _: H;
     let _: d::g::I;
+
+    let _: Arc<AtomicBool>;
+    let _: Ordering;
 }
diff --git a/src/test/ui/use-nested-groups-error.rs b/src/test/ui/use-nested-groups-error.rs
new file mode 100644
index 00000000000..a9b6b3ee70d
--- /dev/null
+++ b/src/test/ui/use-nested-groups-error.rs
@@ -0,0 +1,27 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(use_nested_groups)]
+
+mod a {
+    pub mod b1 {
+        pub enum C2 {}
+    }
+
+    pub enum B2 {}
+}
+
+use a::{b1::{C1, C2}, B2};
+//~^ ERROR unresolved import `a::b1::C1`
+
+fn main() {
+    let _: C2;
+    let _: B2;
+}
diff --git a/src/test/ui/use-nested-groups-error.stderr b/src/test/ui/use-nested-groups-error.stderr
new file mode 100644
index 00000000000..cae34684c8e
--- /dev/null
+++ b/src/test/ui/use-nested-groups-error.stderr
@@ -0,0 +1,8 @@
+error[E0432]: unresolved import `a::b1::C1`
+  --> $DIR/use-nested-groups-error.rs:21:14
+   |
+21 | use a::{b1::{C1, C2}, B2};
+   |              ^^ no `C1` in `a::b1`. Did you mean to use `C2`?
+
+error: aborting due to previous error
+

From 898fdcc3eda5c5fa32e893b27f066d2dad77ea77 Mon Sep 17 00:00:00 2001
From: Marco A L Barbosa <malbarbo@gmail.com>
Date: Wed, 24 Jan 2018 15:11:15 -0200
Subject: [PATCH 22/26] Make run-pass/env-home-dir.rs test more robust

Remove the assumption that home_dir always returns Some

This allows the test to be executed with
[cross](https://github.com/japaric/cross).
---
 src/test/run-pass/env-home-dir.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/run-pass/env-home-dir.rs b/src/test/run-pass/env-home-dir.rs
index 22e440c6ffa..449d3b044e9 100644
--- a/src/test/run-pass/env-home-dir.rs
+++ b/src/test/run-pass/env-home-dir.rs
@@ -16,6 +16,9 @@
 use std::env::*;
 use std::path::PathBuf;
 
+/// When HOME is not set, some platforms return `None`, but others return `Some` with a default.
+/// Just check that it is not "/home/MountainView".
+
 #[cfg(unix)]
 fn main() {
     let oldhome = var("HOME");
@@ -27,7 +30,7 @@ fn main() {
     if cfg!(target_os = "android") {
         assert!(home_dir().is_none());
     } else {
-        assert!(home_dir().is_some());
+        assert_ne!(home_dir(), Some(PathBuf::from("/home/MountainView")));
     }
 }
 

From adeb0aeb4a21fe3a944285ad72dab01c10765398 Mon Sep 17 00:00:00 2001
From: Niko Matsakis <niko@alum.mit.edu>
Date: Mon, 29 Jan 2018 13:28:23 -0500
Subject: [PATCH 23/26] move comment right onto the line in question

---
 src/test/run-pass/env-home-dir.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/run-pass/env-home-dir.rs b/src/test/run-pass/env-home-dir.rs
index 449d3b044e9..9bbff1eeb81 100644
--- a/src/test/run-pass/env-home-dir.rs
+++ b/src/test/run-pass/env-home-dir.rs
@@ -16,9 +16,6 @@
 use std::env::*;
 use std::path::PathBuf;
 
-/// When HOME is not set, some platforms return `None`, but others return `Some` with a default.
-/// Just check that it is not "/home/MountainView".
-
 #[cfg(unix)]
 fn main() {
     let oldhome = var("HOME");
@@ -30,6 +27,9 @@ fn main() {
     if cfg!(target_os = "android") {
         assert!(home_dir().is_none());
     } else {
+        // When HOME is not set, some platforms return `None`,
+        // but others return `Some` with a default.
+        // Just check that it is not "/home/MountainView".
         assert_ne!(home_dir(), Some(PathBuf::from("/home/MountainView")));
     }
 }

From ae98f4cac55af394c3e6477ed4c366f63b6626aa Mon Sep 17 00:00:00 2001
From: Oliver Middleton <olliemail27@gmail.com>
Date: Mon, 29 Jan 2018 16:44:35 +0000
Subject: [PATCH 24/26] rustdoc: Fix link title rendering with hoedown

The link title needs to be HTML escaped.
---
 src/librustdoc/html/markdown.rs       |  2 +-
 src/test/rustdoc/link-title-escape.rs | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 src/test/rustdoc/link-title-escape.rs

diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs
index dce0c4b001a..82ced00644d 100644
--- a/src/librustdoc/html/markdown.rs
+++ b/src/librustdoc/html/markdown.rs
@@ -872,7 +872,7 @@ pub fn render(w: &mut fmt::Formatter,
         let link_out = format!("<a href=\"{link}\"{title}>{content}</a>",
                                link = link_buf,
                                title = title.map_or(String::new(),
-                                                    |t| format!(" title=\"{}\"", t)),
+                                                    |t| format!(" title=\"{}\"", Escape(&t))),
                                content = content.unwrap_or(String::new()));
 
         unsafe { hoedown_buffer_put(ob, link_out.as_ptr(), link_out.len()); }
diff --git a/src/test/rustdoc/link-title-escape.rs b/src/test/rustdoc/link-title-escape.rs
new file mode 100644
index 00000000000..eb53c3c2cb5
--- /dev/null
+++ b/src/test/rustdoc/link-title-escape.rs
@@ -0,0 +1,19 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: -Z unstable-options --disable-commonmark
+
+#![crate_name = "foo"]
+
+//! hello [foo]
+//!
+//! [foo]: url 'title & <stuff> & "things"'
+
+// @has 'foo/index.html' 'title &amp; &lt;stuff&gt; &amp; &quot;things&quot;'

From 2184400be7f6b695792af7ddde14482f3e72f1e1 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Mon, 29 Jan 2018 16:37:57 -0600
Subject: [PATCH 25/26] Update comment

---
 src/libsyntax/ext/tt/macro_parser.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index af18801c97e..1a9849ca530 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -113,7 +113,8 @@ enum TokenTreeOrTokenTreeVec {
 }
 
 impl TokenTreeOrTokenTreeVec {
-    /// Returns the number of constituent token trees of `self`.
+    /// Returns the number of constituent top-level token trees of `self` (top-level in that it
+    /// will not recursively descend into subtrees).
     fn len(&self) -> usize {
         match *self {
             TtSeq(ref v) => v.len(),

From 576294237b10fff22bc462398ff7d06fffa05bd0 Mon Sep 17 00:00:00 2001
From: Mark Mansi <markm@cs.wisc.edu>
Date: Mon, 29 Jan 2018 17:08:04 -0600
Subject: [PATCH 26/26] fix typos

---
 src/libsyntax/ext/tt/quoted.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 61dc3d32f20..c55dfaba8f6 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -37,7 +37,7 @@ impl Delimited {
         token::CloseDelim(self.delim)
     }
 
-    /// Return a `self::TokenTree` witha a `Span` corresponding to the opening delimiter.
+    /// Return a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
     pub fn open_tt(&self, span: Span) -> TokenTree {
         let open_span = if span == DUMMY_SP {
             DUMMY_SP
@@ -47,7 +47,7 @@ impl Delimited {
         TokenTree::Token(open_span, self.open_token())
     }
 
-    /// Return a `self::TokenTree` witha a `Span` corresponding to the closing delimiter.
+    /// Return a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
     pub fn close_tt(&self, span: Span) -> TokenTree {
         let close_span = if span == DUMMY_SP {
             DUMMY_SP
@@ -232,7 +232,7 @@ pub fn parse(
 ///
 /// # Parameters
 ///
-/// - `tree`: the tree wish to convert.
+/// - `tree`: the tree we wish to convert.
 /// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish
 ///   converting `tree`
 /// - `expect_matchers`: same as for `parse` (see above).
@@ -327,7 +327,7 @@ where
 /// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing
 /// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator
 /// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some
-/// stream of tokens in an invokation of a macro.
+/// stream of tokens in an invocation of a macro.
 ///
 /// This function will take some input iterator `input` corresponding to `span` and a parsing
 /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene