OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add UI testing framework

This commit is contained in:
Niko Matsakis 2016-05-11 12:28:52 -07:00
parent 33a5c9dfd1
commit cb112dc8cf
9 changed files with 236 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
mk/tests.mk
View File

@ -274,6 +274,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
check-stage$(1)-T-$(2)-H-$(3)-debuginfo-gdb-exec \
check-stage$(1)-T-$(2)-H-$(3)-debuginfo-lldb-exec \
check-stage$(1)-T-$(2)-H-$(3)-incremental-exec \
check-stage$(1)-T-$(2)-H-$(3)-ui-exec \
check-stage$(1)-T-$(2)-H-$(3)-doc-exec \
check-stage$(1)-T-$(2)-H-$(3)-pretty-exec
@ -452,6 +453,9 @@ CODEGEN_CC := $(call rwildcard,$(S)src/test/codegen/,*.cc)
CODEGEN_UNITS_RS := $(call rwildcard,$(S)src/test/codegen-units/,*.rs)
INCREMENTAL_RS := $(call rwildcard,$(S)src/test/incremental/,*.rs)
RMAKE_RS := $(wildcard $(S)src/test/run-make/*/Makefile)
UI_RS := $(call rwildcard,$(S)src/test/ui/,*.rs) \
$(call rwildcard,$(S)src/test/ui/,*.stdout) \
$(call rwildcard,$(S)src/test/ui/,*.stderr)
RUSTDOCCK_RS := $(call rwildcard,$(S)src/test/rustdoc/,*.rs)
RPASS_TESTS := $(RPASS_RS)
@ -469,6 +473,7 @@ CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
CODEGEN_UNITS_TESTS := $(CODEGEN_UNITS_RS)
INCREMENTAL_TESTS := $(INCREMENTAL_RS)
RMAKE_TESTS := $(RMAKE_RS)
UI_TESTS := $(UI_RS)
RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)
CTEST_SRC_BASE_rpass = run-pass
@ -541,6 +546,11 @@ CTEST_BUILD_BASE_rmake = run-make
CTEST_MODE_rmake = run-make
CTEST_RUNTOOL_rmake = $(CTEST_RUNTOOL)
CTEST_SRC_BASE_ui = ui
CTEST_BUILD_BASE_ui = ui
CTEST_MODE_ui = ui
CTEST_RUNTOOL_ui = $(CTEST_RUNTOOL)
CTEST_SRC_BASE_rustdocck = rustdoc
CTEST_BUILD_BASE_rustdocck = rustdoc
CTEST_MODE_rustdocck = rustdoc
@ -672,7 +682,7 @@ CTEST_DEPS_codegen-units_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_UNITS_TESTS)
CTEST_DEPS_incremental_$(1)-T-$(2)-H-$(3) = $$(INCREMENTAL_TESTS)
CTEST_DEPS_rmake_$(1)-T-$(2)-H-$(3) = $$(RMAKE_TESTS) \
$$(CSREQ$(1)_T_$(3)_H_$(3)) $$(SREQ$(1)_T_$(2)_H_$(3))
CTEST_DEPS_ui_$(1)-T-$(2)-H-$(3) = $$(UI_TESTS)
CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
$(S)src/etc/htmldocck.py
@ -744,7 +754,7 @@ endef
CTEST_NAMES = rpass rpass-valgrind rpass-full rfail-full cfail-full rfail cfail pfail \
debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck incremental \
rmake
rmake ui
$(foreach host,$(CFG_HOST), \
$(eval $(foreach target,$(CFG_TARGET), \
@ -943,6 +953,7 @@ TEST_GROUPS = \
codegen \
codegen-units \
incremental \
ui \
doc \
$(foreach docname,$(DOC_NAMES),doc-$(docname)) \
pretty \

31
src/test/ui/README.md Normal file
View File

@ -0,0 +1,31 @@
# Guide to the UI Tests
The UI tests are intended to capture the compiler's complete output,
so that we can test all aspects of the presentation. They work by
compiling a file (e.g., `hello_world/main.rs`), capturing the output,
and then applying some normalization (see below). This normalized
result is then compared against reference files named
`hello_world/main.stderr` and `hello_world/main.stdout`. If either of
those files doesn't exist, the output must be empty. If the test run
fails, we will print out the current output, but it is also saved in
`build/<target-triple>/test/ui/hello_world/main.stdout` (this path is
printed as part of the test failure mesage), so you can run `diff` and
so forth.
# Editing and updating the reference files
If you have changed the compiler's output intentionally, or you are
making a new test, you can use the script `update-references.sh` to
update the references. When you run the test framework, it will report
various errors: in those errors is a command you can use to run the
`update-references.sh` script, which will then copy over the files
from the build directory and use them as the new reference. You can
also just run `update-all-references.sh`. In both cases, you can run
the script with `--help` to get a help message.
# Normalization
The normalization applied is aimed at filenames:
- the test directory is replaced with `$DIR`
- all backslashes (\) are converted to forward slashes (/) (for windows)

15
src/test/ui/hello_world/main.rs Normal file
View File

@ -0,0 +1,15 @@
// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// Test that compiling hello world succeeds with no output of any kind.
fn main() {
println!("Hello, world!");
}

17
src/test/ui/mismatched_types/main.rs Normal file
View File

@ -0,0 +1,17 @@
// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// rustc-env:RUST_NEW_ERROR_FORMAT
fn main() {
let x: u32 = (
);
}

8
src/test/ui/mismatched_types/main.stderr Normal file
View File

@ -0,0 +1,8 @@
error: mismatched types [--explain E0308]
--> $DIR/main.rs:14:18
14 |> let x: u32 = (
|> ^ expected u32, found ()
note: expected type `u32`
note: found type `()`
error: aborting due to previous error

21
src/test/ui/update-all-references.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
# A script to update the references for all tests. The idea is that
# you do a run, which will generate files in the build directory
# containing the (normalized) actual output of the compiler. You then
# run this script, which will copy those files over. If you find
# yourself manually editing a foo.stderr file, you're doing it wrong.
#
# See all `update-references.sh`, if you just want to update a single test.
if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" != "" ]]; then
echo "usage: $0 <build-directory>"
echo ""
echo "For example:"
echo " $0 ../../../build/x86_64-apple-darwin/test/ui"
fi
BUILD_DIR=$PWD/$1
MY_DIR=$(dirname $0)
cd $MY_DIR
find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR

41
src/test/ui/update-references.sh Executable file
View File

@ -0,0 +1,41 @@
#!/bin/bash
# A script to update the references for particular tests. The idea is
# that you do a run, which will generate files in the build directory
# containing the (normalized) actual output of the compiler. This
# script will then copy that output and replace the "expected output"
# files. You can then commit the changes.
#
# If you find yourself manually editing a foo.stderr file, you're
# doing it wrong.
if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
echo "usage: $0 <build-directory> <relative-path-to-rs-files>"
echo ""
echo "For example:"
echo " $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
fi
MYDIR=$(dirname $0)
BUILD_DIR="$1"
shift
while [[ "$1" != "" ]]; do
STDERR_NAME="${1/%.rs/.stderr}"
STDOUT_NAME="${1/%.rs/.stdout}"
shift
if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME > /dev/null); then
echo updating $MYDIR/$STDOUT_NAME
cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
fi
if [ -f $BUILD_DIR/$STDERR_NAME ] && \
! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME > /dev/null); then
echo updating $MYDIR/$STDERR_NAME
cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
fi
done

3
src/tools/compiletest/src/common.rs
View File

@ -28,6 +28,7 @@ pub enum Mode {
CodegenUnits,
Incremental,
RunMake,
Ui,
}
impl FromStr for Mode {
@ -47,6 +48,7 @@ impl FromStr for Mode {
"codegen-units" => Ok(CodegenUnits),
"incremental" => Ok(Incremental),
"run-make" => Ok(RunMake),
"ui" => Ok(Ui),
_ => Err(()),
}
}
@ -68,6 +70,7 @@ impl fmt::Display for Mode {
CodegenUnits => "codegen-units",
Incremental => "incremental",
RunMake => "run-make",
Ui => "ui",
}, f)
}
}

88
src/tools/compiletest/src/runtest.rs
View File

@ -11,7 +11,7 @@
use common::Config;
use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
use common::{Incremental, RunMake};
use common::{Incremental, RunMake, Ui};
use errors::{self, ErrorKind, Error};
use json;
use header::TestProps;
@ -29,6 +29,7 @@ use std::io::prelude::*;
use std::net::TcpStream;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, ExitStatus};
use std::str;
pub fn run(config: Config, testpaths: &TestPaths) {
match &*config.target {
@ -118,6 +119,7 @@ impl<'test> TestCx<'test> {
CodegenUnits => self.run_codegen_units_test(),
Incremental => self.run_incremental_test(),
RunMake => self.run_rmake_test(),
Ui => self.run_ui_test(),
}
}
@ -1314,6 +1316,7 @@ actual:\n\
Codegen |
Rustdoc |
RunMake |
Ui |
CodegenUnits => {
// do not use JSON output
}
@ -2096,6 +2099,89 @@ actual:\n\
}
fs::remove_dir(path)
}
fn run_ui_test(&self) {
println!("ui: {}", self.testpaths.file.display());
let proc_res = self.compile_test();
let expected_stderr_path = self.expected_output_path("stderr");
let expected_stderr = self.load_expected_output(&expected_stderr_path);
let expected_stdout_path = self.expected_output_path("stdout");
let expected_stdout = self.load_expected_output(&expected_stdout_path);
let normalized_stdout = self.normalize_output(&proc_res.stdout);
let normalized_stderr = self.normalize_output(&proc_res.stderr);
let mut errors = 0;
errors += self.compare_output("stdout", normalized_stdout.as_bytes(), &expected_stdout);
errors += self.compare_output("stderr", normalized_stderr.as_bytes(), &expected_stderr);
if errors > 0 {
println!("To update references, run this command from build directory:");
let relative_path_to_file =
self.testpaths.relative_dir
.join(self.testpaths.file.file_name().unwrap());
println!("{}/update-references.sh '{}' '{}'",
self.config.src_base.display(),
self.config.build_base.display(),
relative_path_to_file.display());
self.fatal(&format!("{} errors occurred comparing output.", errors));
}
}
fn normalize_output(&self, output: &str) -> String {
let parent_dir = self.testpaths.file.parent().unwrap();
let parent_dir_str = parent_dir.display().to_string();
output.replace(&parent_dir_str, "$DIR")
.replace("\\", "/") // windows, you know.
}
fn expected_output_path(&self, kind: &str) -> PathBuf {
let extension = match self.revision {
Some(r) => format!("{}.{}", r, kind),
None => kind.to_string(),
};
self.testpaths.file.with_extension(extension)
}
fn load_expected_output(&self, path: &Path) -> Vec<u8> {
if !path.exists() {
return vec![];
}
let mut result = Vec::new();
match File::open(path).and_then(|mut f| f.read_to_end(&mut result)) {
Ok(_) => result,
Err(e) => {
self.fatal(&format!("failed to load expected output from `{}`: {}", path.display(), e))
}
}
}
fn compare_output(&self, kind: &str, actual: &[u8], expected: &[u8]) -> usize {
if self.config.verbose {
println!("normalized {}:\n{}\n", kind, str::from_utf8(actual).unwrap_or("not utf8"));
println!("expected {}:\n{}\n", kind, str::from_utf8(expected).unwrap_or("not utf8"));
}
if actual == expected {
return 0;
}
let output_file = self.output_base_name().with_extension(kind);
match File::create(&output_file).and_then(|mut f| f.write_all(actual)) {
Ok(()) => { }
Err(e) => {
self.fatal(&format!("failed to write {} to `{}`: {}",
kind, output_file.display(), e))
}
}
println!("\nThe actual {0} differed from the expected {0}.", kind);
println!("Actual {} saved to {}", kind, output_file.display());
1
}
}
struct ProcArgs {