Auto merge of #41824 - Mark-Simulacrum:undo-yacc-removal, r=nagisa

Readd LALR grammar

Reverts a portion of #41705. Please let me know if I missed anything.

r? @nagisa

src/grammar/.gitignore

@@ -0,0 +1,3 @@
+*.class
+*.java
+*.tokens

src/grammar/lexer.l

@@ -0,0 +1,343 @@
+%{
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include <stdio.h>
+#include <ctype.h>
+
+static int num_hashes;
+static int end_hashes;
+static int saw_non_hash;
+
+%}
+
+%option stack
+%option yylineno
+
+%x str
+%x rawstr
+%x rawstr_esc_begin
+%x rawstr_esc_body
+%x rawstr_esc_end
+%x byte
+%x bytestr
+%x rawbytestr
+%x rawbytestr_nohash
+%x pound
+%x shebang_or_attr
+%x ltorchar
+%x linecomment
+%x doc_line
+%x blockcomment
+%x doc_block
+%x suffix
+
+ident [a-zA-Z\x80-\xff_][a-zA-Z0-9\x80-\xff_]*
+
+%%
+
+<suffix>{ident}            { BEGIN(INITIAL); }
+<suffix>(.|\n)  { yyless(0); BEGIN(INITIAL); }
+
+[ \n\t\r]             { }
+
+\xef\xbb\xbf {
+  // UTF-8 byte order mark (BOM), ignore if in line 1, error otherwise
+  if (yyget_lineno() != 1) {
+    return -1;
+  }
+}
+
+\/\/(\/|\!)           { BEGIN(doc_line); yymore(); }
+<doc_line>\n          { BEGIN(INITIAL);
+                        yyleng--;
+                        yytext[yyleng] = 0;
+                        return ((yytext[2] == '!') ? INNER_DOC_COMMENT : OUTER_DOC_COMMENT);
+                      }
+<doc_line>[^\n]*      { yymore(); }
+
+\/\/|\/\/\/\/         { BEGIN(linecomment); }
+<linecomment>\n       { BEGIN(INITIAL); }
+<linecomment>[^\n]*   { }
+
+\/\*(\*|\!)[^*]       { yy_push_state(INITIAL); yy_push_state(doc_block); yymore(); }
+<doc_block>\/\*       { yy_push_state(doc_block); yymore(); }
+<doc_block>\*\/       {
+    yy_pop_state();
+    if (yy_top_state() == doc_block) {
+        yymore();
+    } else {
+        return ((yytext[2] == '!') ? INNER_DOC_COMMENT : OUTER_DOC_COMMENT);
+    }
+}
+<doc_block>(.|\n)     { yymore(); }
+
+\/\*                  { yy_push_state(blockcomment); }
+<blockcomment>\/\*    { yy_push_state(blockcomment); }
+<blockcomment>\*\/    { yy_pop_state(); }
+<blockcomment>(.|\n)   { }
+
+_        { return UNDERSCORE; }
+as       { return AS; }
+box      { return BOX; }
+break    { return BREAK; }
+const    { return CONST; }
+continue { return CONTINUE; }
+crate    { return CRATE; }
+else     { return ELSE; }
+enum     { return ENUM; }
+extern   { return EXTERN; }
+false    { return FALSE; }
+fn       { return FN; }
+for      { return FOR; }
+if       { return IF; }
+impl     { return IMPL; }
+in       { return IN; }
+let      { return LET; }
+loop     { return LOOP; }
+match    { return MATCH; }
+mod      { return MOD; }
+move     { return MOVE; }
+mut      { return MUT; }
+priv     { return PRIV; }
+proc     { return PROC; }
+pub      { return PUB; }
+ref      { return REF; }
+return   { return RETURN; }
+self     { return SELF; }
+static   { return STATIC; }
+struct   { return STRUCT; }
+trait    { return TRAIT; }
+true     { return TRUE; }
+type     { return TYPE; }
+typeof   { return TYPEOF; }
+unsafe   { return UNSAFE; }
+use      { return USE; }
+where    { return WHERE; }
+while    { return WHILE; }
+
+{ident}  { return IDENT; }
+
+0x[0-9a-fA-F_]+                                    { BEGIN(suffix); return LIT_INTEGER; }
+0o[0-8_]+                                          { BEGIN(suffix); return LIT_INTEGER; }
+0b[01_]+                                           { BEGIN(suffix); return LIT_INTEGER; }
+[0-9][0-9_]*                                       { BEGIN(suffix); return LIT_INTEGER; }
+[0-9][0-9_]*\.(\.|[a-zA-Z])    { yyless(yyleng - 2); BEGIN(suffix); return LIT_INTEGER; }
+
+[0-9][0-9_]*\.[0-9_]*([eE][-\+]?[0-9_]+)?          { BEGIN(suffix); return LIT_FLOAT; }
+[0-9][0-9_]*(\.[0-9_]*)?[eE][-\+]?[0-9_]+          { BEGIN(suffix); return LIT_FLOAT; }
+
+;      { return ';'; }
+,      { return ','; }
+\.\.\. { return DOTDOTDOT; }
+\.\.   { return DOTDOT; }
+\.     { return '.'; }
+\(     { return '('; }
+\)     { return ')'; }
+\{     { return '{'; }
+\}     { return '}'; }
+\[     { return '['; }
+\]     { return ']'; }
+@      { return '@'; }
+#      { BEGIN(pound); yymore(); }
+<pound>\! { BEGIN(shebang_or_attr); yymore(); }
+<shebang_or_attr>\[ {
+  BEGIN(INITIAL);
+  yyless(2);
+  return SHEBANG;
+}
+<shebang_or_attr>[^\[\n]*\n {
+  // Since the \n was eaten as part of the token, yylineno will have
+  // been incremented to the value 2 if the shebang was on the first
+  // line. This yyless undoes that, setting yylineno back to 1.
+  yyless(yyleng - 1);
+  if (yyget_lineno() == 1) {
+    BEGIN(INITIAL);
+    return SHEBANG_LINE;
+  } else {
+    BEGIN(INITIAL);
+    yyless(2);
+    return SHEBANG;
+  }
+}
+<pound>. { BEGIN(INITIAL); yyless(1); return '#'; }
+
+\~     { return '~'; }
+::     { return MOD_SEP; }
+:      { return ':'; }
+\$     { return '$'; }
+\?     { return '?'; }
+
+==    { return EQEQ; }
+=>    { return FAT_ARROW; }
+=     { return '='; }
+\!=   { return NE; }
+\!    { return '!'; }
+\<=   { return LE; }
+\<\<  { return SHL; }
+\<\<= { return SHLEQ; }
+\<    { return '<'; }
+\>=   { return GE; }
+\>\>  { return SHR; }
+\>\>= { return SHREQ; }
+\>    { return '>'; }
+
+\x27                                  { BEGIN(ltorchar); yymore(); }
+<ltorchar>static                      { BEGIN(INITIAL); return STATIC_LIFETIME; }
+<ltorchar>{ident}                     { BEGIN(INITIAL); return LIFETIME; }
+<ltorchar>\\[nrt\\\x27\x220]\x27      { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>\\x[0-9a-fA-F]{2}\x27       { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>\\u\{[0-9a-fA-F]?{6}\}\x27  { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>.\x27                       { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>[\x80-\xff]{2,4}\x27        { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar><<EOF>>                     { BEGIN(INITIAL); return -1; }
+
+b\x22              { BEGIN(bytestr); yymore(); }
+<bytestr>\x22      { BEGIN(suffix); return LIT_BYTE_STR; }
+
+<bytestr><<EOF>>                { return -1; }
+<bytestr>\\[n\nrt\\\x27\x220]   { yymore(); }
+<bytestr>\\x[0-9a-fA-F]{2}      { yymore(); }
+<bytestr>\\u\{[0-9a-fA-F]?{6}\} { yymore(); }
+<bytestr>\\[^n\nrt\\\x27\x220]  { return -1; }
+<bytestr>(.|\n)                 { yymore(); }
+
+br\x22                      { BEGIN(rawbytestr_nohash); yymore(); }
+<rawbytestr_nohash>\x22     { BEGIN(suffix); return LIT_BYTE_STR_RAW; }
+<rawbytestr_nohash>(.|\n)   { yymore(); }
+<rawbytestr_nohash><<EOF>>  { return -1; }
+
+br/# {
+    BEGIN(rawbytestr);
+    yymore();
+    num_hashes = 0;
+    saw_non_hash = 0;
+    end_hashes = 0;
+}
+<rawbytestr># {
+    if (!saw_non_hash) {
+        num_hashes++;
+    } else if (end_hashes != 0) {
+        end_hashes++;
+        if (end_hashes == num_hashes) {
+            BEGIN(INITIAL);
+            return LIT_BYTE_STR_RAW;
+        }
+    }
+    yymore();
+}
+<rawbytestr>\x22# {
+    end_hashes = 1;
+    if (end_hashes == num_hashes) {
+        BEGIN(INITIAL);
+        return LIT_BYTE_STR_RAW;
+    }
+    yymore();
+}
+<rawbytestr>(.|\n) {
+    if (!saw_non_hash) {
+        saw_non_hash = 1;
+    }
+    if (end_hashes != 0) {
+        end_hashes = 0;
+    }
+    yymore();
+}
+<rawbytestr><<EOF>> { return -1; }
+
+b\x27                        { BEGIN(byte); yymore(); }
+<byte>\\[nrt\\\x27\x220]\x27 { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>\\x[0-9a-fA-F]{2}\x27  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>\\u[0-9a-fA-F]{4}\x27  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>\\U[0-9a-fA-F]{8}\x27  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>.\x27                  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte><<EOF>>                { BEGIN(INITIAL); return -1; }
+
+r\x22           { BEGIN(rawstr); yymore(); }
+<rawstr>\x22    { BEGIN(suffix); return LIT_STR_RAW; }
+<rawstr>(.|\n)  { yymore(); }
+<rawstr><<EOF>> { return -1; }
+
+r/#             {
+    BEGIN(rawstr_esc_begin);
+    yymore();
+    num_hashes = 0;
+    saw_non_hash = 0;
+    end_hashes = 0;
+}
+
+<rawstr_esc_begin># {
+    num_hashes++;
+    yymore();
+}
+<rawstr_esc_begin>\x22 {
+    BEGIN(rawstr_esc_body);
+    yymore();
+}
+<rawstr_esc_begin>(.|\n) { return -1; }
+
+<rawstr_esc_body>\x22/# {
+  BEGIN(rawstr_esc_end);
+  yymore();
+ }
+<rawstr_esc_body>(.|\n) {
+  yymore();
+ }
+
+<rawstr_esc_end># {
+  end_hashes++;
+  if (end_hashes == num_hashes) {
+    BEGIN(INITIAL);
+    return LIT_STR_RAW;
+  }
+  yymore();
+ }
+<rawstr_esc_end>[^#] {
+  end_hashes = 0;
+  BEGIN(rawstr_esc_body);
+  yymore();
+ }
+
+<rawstr_esc_begin,rawstr_esc_body,rawstr_esc_end><<EOF>> { return -1; }
+
+\x22                     { BEGIN(str); yymore(); }
+<str>\x22                { BEGIN(suffix); return LIT_STR; }
+
+<str><<EOF>>                { return -1; }
+<str>\\[n\nr\rt\\\x27\x220] { yymore(); }
+<str>\\x[0-9a-fA-F]{2}      { yymore(); }
+<str>\\u\{[0-9a-fA-F]?{6}\} { yymore(); }
+<str>\\[^n\nrt\\\x27\x220]  { return -1; }
+<str>(.|\n)                 { yymore(); }
+
+\<-  { return LARROW; }
+-\>  { return RARROW; }
+-    { return '-'; }
+-=   { return MINUSEQ; }
+&&   { return ANDAND; }
+&    { return '&'; }
+&=   { return ANDEQ; }
+\|\| { return OROR; }
+\|   { return '|'; }
+\|=  { return OREQ; }
+\+   { return '+'; }
+\+=  { return PLUSEQ; }
+\*   { return '*'; }
+\*=  { return STAREQ; }
+\/   { return '/'; }
+\/=  { return SLASHEQ; }
+\^   { return '^'; }
+\^=  { return CARETEQ; }
+%    { return '%'; }
+%=   { return PERCENTEQ; }
+
+<<EOF>> { return 0; }
+
+%%

src/grammar/parser-lalr-main.c

@@ -0,0 +1,203 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+extern int yylex();
+extern int rsparse();
+
+#define PUSHBACK_LEN 4
+
+static char pushback[PUSHBACK_LEN];
+static int verbose;
+
+void print(const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  if (verbose) {
+    vprintf(format, args);
+  }
+  va_end(args);
+}
+
+// If there is a non-null char at the head of the pushback queue,
+// dequeue it and shift the rest of the queue forwards. Otherwise,
+// return the token from calling yylex.
+int rslex() {
+  if (pushback[0] == '\0') {
+    return yylex();
+  } else {
+    char c = pushback[0];
+    memmove(pushback, pushback + 1, PUSHBACK_LEN - 1);
+    pushback[PUSHBACK_LEN - 1] = '\0';
+    return c;
+  }
+}
+
+// Note: this does nothing if the pushback queue is full. As long as
+// there aren't more than PUSHBACK_LEN consecutive calls to push_back
+// in an action, this shouldn't be a problem.
+void push_back(char c) {
+  for (int i = 0; i < PUSHBACK_LEN; ++i) {
+    if (pushback[i] == '\0') {
+      pushback[i] = c;
+      break;
+    }
+  }
+}
+
+extern int rsdebug;
+
+struct node {
+  struct node *next;
+  struct node *prev;
+  int own_string;
+  char const *name;
+  int n_elems;
+  struct node *elems[];
+};
+
+struct node *nodes = NULL;
+int n_nodes;
+
+struct node *mk_node(char const *name, int n, ...) {
+  va_list ap;
+  int i = 0;
+  unsigned sz = sizeof(struct node) + (n * sizeof(struct node *));
+  struct node *nn, *nd = (struct node *)malloc(sz);
+
+  print("# New %d-ary node: %s = %p\n", n, name, nd);
+
+  nd->own_string = 0;
+  nd->prev = NULL;
+  nd->next = nodes;
+  if (nodes) {
+    nodes->prev = nd;
+  }
+  nodes = nd;
+
+  nd->name = name;
+  nd->n_elems = n;
+
+  va_start(ap, n);
+  while (i < n) {
+    nn = va_arg(ap, struct node *);
+    print("#   arg[%d]: %p\n", i, nn);
+    print("#            (%s ...)\n", nn->name);
+    nd->elems[i++] = nn;
+  }
+  va_end(ap);
+  n_nodes++;
+  return nd;
+}
+
+struct node *mk_atom(char *name) {
+  struct node *nd = mk_node((char const *)strdup(name), 0);
+  nd->own_string = 1;
+  return nd;
+}
+
+struct node *mk_none() {
+  return mk_atom("<none>");
+}
+
+struct node *ext_node(struct node *nd, int n, ...) {
+  va_list ap;
+  int i = 0, c = nd->n_elems + n;
+  unsigned sz = sizeof(struct node) + (c * sizeof(struct node *));
+  struct node *nn;
+
+  print("# Extending %d-ary node by %d nodes: %s = %p",
+        nd->n_elems, c, nd->name, nd);
+
+  if (nd->next) {
+    nd->next->prev = nd->prev;
+  }
+  if (nd->prev) {
+    nd->prev->next = nd->next;
+  }
+  nd = realloc(nd, sz);
+  nd->prev = NULL;
+  nd->next = nodes;
+  nodes->prev = nd;
+  nodes = nd;
+
+  print(" ==> %p\n", nd);
+
+  va_start(ap, n);
+  while (i < n) {
+    nn = va_arg(ap, struct node *);
+    print("#   arg[%d]: %p\n", i, nn);
+    print("#            (%s ...)\n", nn->name);
+    nd->elems[nd->n_elems++] = nn;
+    ++i;
+  }
+  va_end(ap);
+  return nd;
+}
+
+int const indent_step = 4;
+
+void print_indent(int depth) {
+  while (depth) {
+    if (depth-- % indent_step == 0) {
+      print("|");
+    } else {
+      print(" ");
+    }
+  }
+}
+
+void print_node(struct node *n, int depth) {
+  int i = 0;
+  print_indent(depth);
+  if (n->n_elems == 0) {
+    print("%s\n", n->name);
+  } else {
+    print("(%s\n", n->name);
+    for (i = 0; i < n->n_elems; ++i) {
+      print_node(n->elems[i], depth + indent_step);
+    }
+    print_indent(depth);
+    print(")\n");
+  }
+}
+
+int main(int argc, char **argv) {
+  if (argc == 2 && strcmp(argv[1], "-v") == 0) {
+    verbose = 1;
+  } else {
+    verbose = 0;
+  }
+  int ret = 0;
+  struct node *tmp;
+  memset(pushback, '\0', PUSHBACK_LEN);
+  ret = rsparse();
+  print("--- PARSE COMPLETE: ret:%d, n_nodes:%d ---\n", ret, n_nodes);
+  if (nodes) {
+    print_node(nodes, 0);
+  }
+  while (nodes) {
+    tmp = nodes;
+    nodes = tmp->next;
+    if (tmp->own_string) {
+      free((void*)tmp->name);
+    }
+    free(tmp);
+  }
+  return ret;
+}
+
+void rserror(char const *s) {
+  fprintf(stderr, "%s\n", s);
+}

src/grammar/raw-string-literal-ambiguity.md

@@ -0,0 +1,64 @@
+Rust's lexical grammar is not context-free. Raw string literals are the source
+of the problem. Informally, a raw string literal is an `r`, followed by `N`
+hashes (where N can be zero), a quote, any characters, then a quote followed
+by `N` hashes. Critically, once inside the first pair of quotes,
+another quote cannot be followed by `N` consecutive hashes. e.g.
+`r###""###"###` is invalid.
+
+This grammar describes this as best possible:
+
+    R -> 'r' S
+    S -> '"' B '"'
+    S -> '#' S '#'
+    B -> . B
+    B -> ε
+
+Where `.` represents any character, and `ε` the empty string. Consider the
+string `r#""#"#`. This string is not a valid raw string literal, but can be
+accepted as one by the above grammar, using the derivation:
+
+    R : #""#"#
+    S : ""#"
+    S : "#
+    B : #
+    B : ε
+
+(Where `T : U` means the rule `T` is applied, and `U` is the remainder of the
+string.) The difficulty arises from the fact that it is fundamentally
+context-sensitive. In particular, the context needed is the number of hashes.
+
+To prove that Rust's string literals are not context-free, we will use
+the fact that context-free languages are closed under intersection with
+regular languages, and the
+[pumping lemma for context-free languages](https://en.wikipedia.org/wiki/Pumping_lemma_for_context-free_languages).
+
+Consider the regular language `R = r#+""#*"#+`. If Rust's raw string literals are
+context-free, then their intersection with `R`, `R'`, should also be context-free.
+Therefore, to prove that raw string literals are not context-free,
+it is sufficient to prove that `R'` is not context-free.
+
+The language `R'` is `{r#^n""#^m"#^n | m < n}`.
+
+Assume `R'` *is* context-free. Then `R'` has some pumping length `p > 0` for which
+the pumping lemma applies. Consider the following string `s` in `R'`:
+
+`r#^p""#^{p-1}"#^p`
+
+e.g. for `p = 2`: `s = r##""#"##`
+
+Then `s = uvwxy` for some choice of `uvwxy` such that `vx` is non-empty,
+`|vwx| < p+1`, and `uv^iwx^iy` is in `R'` for all `i >= 0`.
+
+Neither `v` nor `x` can contain a `"` or `r`, as the number of these characters
+in any string in `R'` is fixed. So `v` and `x` contain only hashes.
+Consequently, of the three sequences of hashes, `v` and `x` combined
+can only pump two of them.
+If we ever choose the central sequence of hashes, then one of the outer sequences
+will not grow when we pump, leading to an imbalance between the outer sequences.
+Therefore, we must pump both outer sequences of hashes. However,
+there are `p+2` characters between these two sequences of hashes, and `|vwx|` must
+be less than `p+1`. Therefore we have a contradiction, and `R'` must not be
+context-free.
+
+Since `R'` is not context-free, it follows that the Rust's raw string literals
+must not be context-free.

src/grammar/testparser.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# ignore-tidy-linelength
+
+import sys
+
+import os
+import subprocess
+import argparse
+
+# usage: testparser.py [-h] [-p PARSER [PARSER ...]] -s SOURCE_DIR
+
+# Parsers should read from stdin and return exit status 0 for a
+# successful parse, and nonzero for an unsuccessful parse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-p', '--parser', nargs='+')
+parser.add_argument('-s', '--source-dir', nargs=1, required=True)
+args = parser.parse_args(sys.argv[1:])
+
+total = 0
+ok = {}
+bad = {}
+for parser in args.parser:
+    ok[parser] = 0
+    bad[parser] = []
+devnull = open(os.devnull, 'w')
+print("\n")
+
+for base, dirs, files in os.walk(args.source_dir[0]):
+    for f in filter(lambda p: p.endswith('.rs'), files):
+        p = os.path.join(base, f)
+        parse_fail = 'parse-fail' in p
+        if sys.version_info.major == 3:
+            lines = open(p, encoding='utf-8').readlines()
+        else:
+            lines = open(p).readlines()
+        if any('ignore-test' in line or 'ignore-lexer-test' in line for line in lines):
+            continue
+        total += 1
+        for parser in args.parser:
+            if subprocess.call(parser, stdin=open(p), stderr=subprocess.STDOUT, stdout=devnull) == 0:
+                if parse_fail:
+                    bad[parser].append(p)
+                else:
+                    ok[parser] += 1
+            else:
+                if parse_fail:
+                    ok[parser] += 1
+                else:
+                    bad[parser].append(p)
+        parser_stats = ', '.join(['{}: {}'.format(parser, ok[parser]) for parser in args.parser])
+        sys.stdout.write("\033[K\r total: {}, {}, scanned {}"
+                         .format(total, os.path.relpath(parser_stats), os.path.relpath(p)))
+
+devnull.close()
+
+print("\n")
+
+for parser in args.parser:
+    filename = os.path.basename(parser) + '.bad'
+    print("writing {} files that did not yield the correct result with {} to {}".format(len(bad[parser]), parser, filename))
+    with open(filename, "w") as f:
+        for p in bad[parser]:
+            f.write(p)
+            f.write("\n")

src/grammar/tokens.h

@@ -0,0 +1,91 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+enum Token {
+  SHL = 257, // Parser generators reserve 0-256 for char literals
+  SHR,
+  LE,
+  EQEQ,
+  NE,
+  GE,
+  ANDAND,
+  OROR,
+  SHLEQ,
+  SHREQ,
+  MINUSEQ,
+  ANDEQ,
+  OREQ,
+  PLUSEQ,
+  STAREQ,
+  SLASHEQ,
+  CARETEQ,
+  PERCENTEQ,
+  DOTDOT,
+  DOTDOTDOT,
+  MOD_SEP,
+  RARROW,
+  FAT_ARROW,
+  LIT_BYTE,
+  LIT_CHAR,
+  LIT_INTEGER,
+  LIT_FLOAT,
+  LIT_STR,
+  LIT_STR_RAW,
+  LIT_BYTE_STR,
+  LIT_BYTE_STR_RAW,
+  IDENT,
+  UNDERSCORE,
+  LIFETIME,
+
+  // keywords
+  SELF,
+  STATIC,
+  AS,
+  BREAK,
+  CRATE,
+  ELSE,
+  ENUM,
+  EXTERN,
+  FALSE,
+  FN,
+  FOR,
+  IF,
+  IMPL,
+  IN,
+  LET,
+  LOOP,
+  MATCH,
+  MOD,
+  MOVE,
+  MUT,
+  PRIV,
+  PUB,
+  REF,
+  RETURN,
+  STRUCT,
+  TRUE,
+  TRAIT,
+  TYPE,
+  UNSAFE,
+  USE,
+  WHILE,
+  CONTINUE,
+  PROC,
+  BOX,
+  CONST,
+  WHERE,
+  TYPEOF,
+  INNER_DOC_COMMENT,
+  OUTER_DOC_COMMENT,
+
+  SHEBANG,
+  SHEBANG_LINE,
+  STATIC_LIFETIME
+};