Auto merge of #37607 - dns2utf8:doc_grammar, r=alexcrichton

Fix grammar verification

 * Use make check-lexer to verify the grammar.
 * Extend grammar/README
 * Add make clean-grammar rule
 * Add target check-build-lexer-verifier to make tidy, so it will build the verifier with every build and catch future errors

This is the continuation of #34994

r? @steveklabnik @jonathandturner @alexcrichton
This commit is contained in:
bors 2016-11-16 21:02:55 -08:00 committed by GitHub
commit 6cd5be81cc
7 changed files with 37 additions and 12 deletions

6
configure vendored
View File

@ -854,6 +854,12 @@ probe_need CFG_CMAKE cmake
# probe for it only in this case.
if [ -n "$CFG_ANTLR4" ]
then
CFG_ANTLR4_JAR="\"$(find /usr/ -name antlr-complete.jar 2>/dev/null | head -n 1)\""
if [ "x" -eq "x$CFG_ANTLR4_JAR" ]
then
CFG_ANTLR4_JAR="\"$(find ~ -name antlr-complete.jar 2>/dev/null | head -n 1)\""
fi
putvar CFG_ANTLR4_JAR $CFG_ANTLR4_JAR
probe CFG_JAVAC javac
fi

View File

@ -35,7 +35,7 @@ clean-all: clean clean-llvm
clean-llvm: $(CLEAN_LLVM_RULES)
clean: clean-misc $(CLEAN_STAGE_RULES)
clean: clean-misc clean-grammar $(CLEAN_STAGE_RULES)
clean-misc:
@$(call E, cleaning)
@ -47,6 +47,9 @@ clean-misc:
$(Q)rm -Rf dist/*
$(Q)rm -Rf doc
clean-grammar:
@$(call E, cleaning grammar verification)
$(Q)rm -Rf grammar
define CLEAN_GENERIC
clean-generic-$(2)-$(1):

View File

@ -37,7 +37,7 @@ $(BG):
$(BG)RustLexer.class: $(BG) $(SG)RustLexer.g4
$(Q)$(CFG_ANTLR4) -o $(BG) $(SG)RustLexer.g4
$(Q)$(CFG_JAVAC) -d $(BG) $(BG)RustLexer.java
$(Q)$(CFG_JAVAC) -d $(BG) -classpath $(CFG_ANTLR4_JAR) $(BG)RustLexer.java
check-build-lexer-verifier: $(BG)verify

View File

@ -243,7 +243,8 @@ cleantestlibs:
.PHONY: tidy
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
$(SNAPSHOT_RUSTC_POST_CLEANUP)
$(SNAPSHOT_RUSTC_POST_CLEANUP) \
check-build-lexer-verifier
$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src
$(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \

View File

@ -1,14 +1,18 @@
Reference grammar.
# Reference grammar.
Uses [antlr4](http://www.antlr.org/) and a custom Rust tool to compare
ASTs/token streams generated. You can use the `check-lexer` make target to
ASTs/token streams generated. You can use the `make check-lexer` target to
run all of the available tests.
To use manually:
The build of the rust part is included with `make tidy` and can be run with `make check-build-lexer-verifier`.
# Manual build
To use manually, assuming antlr4 ist installed at `/usr/share/java/antlr-complete.jar`:
```
antlr4 RustLexer.g4
javac *.java
javac -classpath /usr/share/java/antlr-complete.jar *.java
rustc -O verify.rs
for file in ../*/**.rs; do
echo $file;
@ -18,3 +22,12 @@ done
Note That the `../*/**.rs` glob will match every `*.rs` file in the above
directory and all of its recursive children. This is a zsh extension.
## Cleanup
To cleanup you can use a command like this:
```bash
rm -f verify *.class *.java *.tokens
```

View File

@ -20,11 +20,11 @@ skipped=0
check() {
grep --silent "// ignore-lexer-test" "$1";
# if it's *not* found...
# if it is *not* found...
if [ $? -eq 1 ]; then
cd $2 # This `cd` is so java will pick up RustLexer.class. I couldn't
cd $2 # This `cd` is so java will pick up RustLexer.class. I could not
# figure out how to wrangle the CLASSPATH, just adding build/grammar
# didn't seem to have any effect.
# did not seem to have any effect.
if $3 RustLexer tokens -tokens < $1 | $4 $1 $5; then
echo "pass: $1"
passed=`expr $passed + 1`

View File

@ -11,6 +11,7 @@
#![feature(plugin, rustc_private)]
extern crate syntax;
extern crate syntax_pos;
extern crate rustc;
#[macro_use]
@ -290,9 +291,10 @@ fn main() {
let options = config::basic_options();
let session = session::build_session(options, &DepGraph::new(false), None,
syntax::diagnostics::registry::Registry::new(&[]),
syntax::errors::registry::Registry::new(&[]),
Rc::new(DummyCrateStore));
let filemap = session.parse_sess.codemap().new_filemap(String::from("<n/a>"), code);
let filemap = session.parse_sess.codemap()
.new_filemap("<n/a>".to_string(), None, code);
let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
let cm = session.codemap();