From cb112dc8cfe105b9eec2a3e1b59dd66eca386ee6 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 11 May 2016 12:28:52 -0700 Subject: [PATCH] add UI testing framework --- mk/tests.mk | 15 +++- src/test/ui/README.md | 31 +++++++++ src/test/ui/hello_world/main.rs | 15 ++++ src/test/ui/mismatched_types/main.rs | 17 +++++ src/test/ui/mismatched_types/main.stderr | 8 +++ src/test/ui/update-all-references.sh | 21 ++++++ src/test/ui/update-references.sh | 41 +++++++++++ src/tools/compiletest/src/common.rs | 3 + src/tools/compiletest/src/runtest.rs | 88 +++++++++++++++++++++++- 9 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 src/test/ui/README.md create mode 100644 src/test/ui/hello_world/main.rs create mode 100644 src/test/ui/mismatched_types/main.rs create mode 100644 src/test/ui/mismatched_types/main.stderr create mode 100755 src/test/ui/update-all-references.sh create mode 100755 src/test/ui/update-references.sh diff --git a/mk/tests.mk b/mk/tests.mk index ea610f63dbf..f9ab84e3f8c 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -274,6 +274,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \ check-stage$(1)-T-$(2)-H-$(3)-debuginfo-gdb-exec \ check-stage$(1)-T-$(2)-H-$(3)-debuginfo-lldb-exec \ check-stage$(1)-T-$(2)-H-$(3)-incremental-exec \ + check-stage$(1)-T-$(2)-H-$(3)-ui-exec \ check-stage$(1)-T-$(2)-H-$(3)-doc-exec \ check-stage$(1)-T-$(2)-H-$(3)-pretty-exec @@ -452,6 +453,9 @@ CODEGEN_CC := $(call rwildcard,$(S)src/test/codegen/,*.cc) CODEGEN_UNITS_RS := $(call rwildcard,$(S)src/test/codegen-units/,*.rs) INCREMENTAL_RS := $(call rwildcard,$(S)src/test/incremental/,*.rs) RMAKE_RS := $(wildcard $(S)src/test/run-make/*/Makefile) +UI_RS := $(call rwildcard,$(S)src/test/ui/,*.rs) \ + $(call rwildcard,$(S)src/test/ui/,*.stdout) \ + $(call rwildcard,$(S)src/test/ui/,*.stderr) RUSTDOCCK_RS := $(call rwildcard,$(S)src/test/rustdoc/,*.rs) RPASS_TESTS := $(RPASS_RS) @@ -469,6 +473,7 @@ CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC) CODEGEN_UNITS_TESTS := $(CODEGEN_UNITS_RS) INCREMENTAL_TESTS := $(INCREMENTAL_RS) RMAKE_TESTS := $(RMAKE_RS) +UI_TESTS := $(UI_RS) RUSTDOCCK_TESTS := $(RUSTDOCCK_RS) CTEST_SRC_BASE_rpass = run-pass @@ -541,6 +546,11 @@ CTEST_BUILD_BASE_rmake = run-make CTEST_MODE_rmake = run-make CTEST_RUNTOOL_rmake = $(CTEST_RUNTOOL) +CTEST_SRC_BASE_ui = ui +CTEST_BUILD_BASE_ui = ui +CTEST_MODE_ui = ui +CTEST_RUNTOOL_ui = $(CTEST_RUNTOOL) + CTEST_SRC_BASE_rustdocck = rustdoc CTEST_BUILD_BASE_rustdocck = rustdoc CTEST_MODE_rustdocck = rustdoc @@ -672,7 +682,7 @@ CTEST_DEPS_codegen-units_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_UNITS_TESTS) CTEST_DEPS_incremental_$(1)-T-$(2)-H-$(3) = $$(INCREMENTAL_TESTS) CTEST_DEPS_rmake_$(1)-T-$(2)-H-$(3) = $$(RMAKE_TESTS) \ $$(CSREQ$(1)_T_$(3)_H_$(3)) $$(SREQ$(1)_T_$(2)_H_$(3)) - +CTEST_DEPS_ui_$(1)-T-$(2)-H-$(3) = $$(UI_TESTS) CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \ $$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \ $(S)src/etc/htmldocck.py @@ -744,7 +754,7 @@ endef CTEST_NAMES = rpass rpass-valgrind rpass-full rfail-full cfail-full rfail cfail pfail \ debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck incremental \ - rmake + rmake ui $(foreach host,$(CFG_HOST), \ $(eval $(foreach target,$(CFG_TARGET), \ @@ -943,6 +953,7 @@ TEST_GROUPS = \ codegen \ codegen-units \ incremental \ + ui \ doc \ $(foreach docname,$(DOC_NAMES),doc-$(docname)) \ pretty \ diff --git a/src/test/ui/README.md b/src/test/ui/README.md new file mode 100644 index 00000000000..dcdeabd8032 --- /dev/null +++ b/src/test/ui/README.md @@ -0,0 +1,31 @@ +# Guide to the UI Tests + +The UI tests are intended to capture the compiler's complete output, +so that we can test all aspects of the presentation. They work by +compiling a file (e.g., `hello_world/main.rs`), capturing the output, +and then applying some normalization (see below). This normalized +result is then compared against reference files named +`hello_world/main.stderr` and `hello_world/main.stdout`. If either of +those files doesn't exist, the output must be empty. If the test run +fails, we will print out the current output, but it is also saved in +`build//test/ui/hello_world/main.stdout` (this path is +printed as part of the test failure mesage), so you can run `diff` and +so forth. + +# Editing and updating the reference files + +If you have changed the compiler's output intentionally, or you are +making a new test, you can use the script `update-references.sh` to +update the references. When you run the test framework, it will report +various errors: in those errors is a command you can use to run the +`update-references.sh` script, which will then copy over the files +from the build directory and use them as the new reference. You can +also just run `update-all-references.sh`. In both cases, you can run +the script with `--help` to get a help message. + +# Normalization + +The normalization applied is aimed at filenames: + +- the test directory is replaced with `$DIR` +- all backslashes (\) are converted to forward slashes (/) (for windows) diff --git a/src/test/ui/hello_world/main.rs b/src/test/ui/hello_world/main.rs new file mode 100644 index 00000000000..61183975577 --- /dev/null +++ b/src/test/ui/hello_world/main.rs @@ -0,0 +1,15 @@ +// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// Test that compiling hello world succeeds with no output of any kind. + +fn main() { + println!("Hello, world!"); +} diff --git a/src/test/ui/mismatched_types/main.rs b/src/test/ui/mismatched_types/main.rs new file mode 100644 index 00000000000..85d9fa53fcf --- /dev/null +++ b/src/test/ui/mismatched_types/main.rs @@ -0,0 +1,17 @@ +// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// rustc-env:RUST_NEW_ERROR_FORMAT + +fn main() { + let x: u32 = ( + ); +} + diff --git a/src/test/ui/mismatched_types/main.stderr b/src/test/ui/mismatched_types/main.stderr new file mode 100644 index 00000000000..98bc11752e0 --- /dev/null +++ b/src/test/ui/mismatched_types/main.stderr @@ -0,0 +1,8 @@ +error: mismatched types [--explain E0308] + --> $DIR/main.rs:14:18 +14 |> let x: u32 = ( + |> ^ expected u32, found () +note: expected type `u32` +note: found type `()` + +error: aborting due to previous error diff --git a/src/test/ui/update-all-references.sh b/src/test/ui/update-all-references.sh new file mode 100755 index 00000000000..cae2a2dba4c --- /dev/null +++ b/src/test/ui/update-all-references.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# A script to update the references for all tests. The idea is that +# you do a run, which will generate files in the build directory +# containing the (normalized) actual output of the compiler. You then +# run this script, which will copy those files over. If you find +# yourself manually editing a foo.stderr file, you're doing it wrong. +# +# See all `update-references.sh`, if you just want to update a single test. + +if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" != "" ]]; then + echo "usage: $0 " + echo "" + echo "For example:" + echo " $0 ../../../build/x86_64-apple-darwin/test/ui" +fi + +BUILD_DIR=$PWD/$1 +MY_DIR=$(dirname $0) +cd $MY_DIR +find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR diff --git a/src/test/ui/update-references.sh b/src/test/ui/update-references.sh new file mode 100755 index 00000000000..703f3f3342b --- /dev/null +++ b/src/test/ui/update-references.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# A script to update the references for particular tests. The idea is +# that you do a run, which will generate files in the build directory +# containing the (normalized) actual output of the compiler. This +# script will then copy that output and replace the "expected output" +# files. You can then commit the changes. +# +# If you find yourself manually editing a foo.stderr file, you're +# doing it wrong. + +if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then + echo "usage: $0 " + echo "" + echo "For example:" + echo " $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs" +fi + +MYDIR=$(dirname $0) + +BUILD_DIR="$1" +shift + +while [[ "$1" != "" ]]; do + STDERR_NAME="${1/%.rs/.stderr}" + STDOUT_NAME="${1/%.rs/.stdout}" + shift + if [ -f $BUILD_DIR/$STDOUT_NAME ] && \ + ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME > /dev/null); then + echo updating $MYDIR/$STDOUT_NAME + cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME + fi + + if [ -f $BUILD_DIR/$STDERR_NAME ] && \ + ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME > /dev/null); then + echo updating $MYDIR/$STDERR_NAME + cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME + fi +done + + diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index ae8beb83530..5ec62e06e37 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -28,6 +28,7 @@ pub enum Mode { CodegenUnits, Incremental, RunMake, + Ui, } impl FromStr for Mode { @@ -47,6 +48,7 @@ impl FromStr for Mode { "codegen-units" => Ok(CodegenUnits), "incremental" => Ok(Incremental), "run-make" => Ok(RunMake), + "ui" => Ok(Ui), _ => Err(()), } } @@ -68,6 +70,7 @@ impl fmt::Display for Mode { CodegenUnits => "codegen-units", Incremental => "incremental", RunMake => "run-make", + Ui => "ui", }, f) } } diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index aa1b9d2bafb..9e4f331d16e 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -11,7 +11,7 @@ use common::Config; use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind}; use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits}; -use common::{Incremental, RunMake}; +use common::{Incremental, RunMake, Ui}; use errors::{self, ErrorKind, Error}; use json; use header::TestProps; @@ -29,6 +29,7 @@ use std::io::prelude::*; use std::net::TcpStream; use std::path::{Path, PathBuf}; use std::process::{Command, Output, ExitStatus}; +use std::str; pub fn run(config: Config, testpaths: &TestPaths) { match &*config.target { @@ -118,6 +119,7 @@ impl<'test> TestCx<'test> { CodegenUnits => self.run_codegen_units_test(), Incremental => self.run_incremental_test(), RunMake => self.run_rmake_test(), + Ui => self.run_ui_test(), } } @@ -1314,6 +1316,7 @@ actual:\n\ Codegen | Rustdoc | RunMake | + Ui | CodegenUnits => { // do not use JSON output } @@ -2096,6 +2099,89 @@ actual:\n\ } fs::remove_dir(path) } + + fn run_ui_test(&self) { + println!("ui: {}", self.testpaths.file.display()); + + let proc_res = self.compile_test(); + + let expected_stderr_path = self.expected_output_path("stderr"); + let expected_stderr = self.load_expected_output(&expected_stderr_path); + + let expected_stdout_path = self.expected_output_path("stdout"); + let expected_stdout = self.load_expected_output(&expected_stdout_path); + + let normalized_stdout = self.normalize_output(&proc_res.stdout); + let normalized_stderr = self.normalize_output(&proc_res.stderr); + + let mut errors = 0; + errors += self.compare_output("stdout", normalized_stdout.as_bytes(), &expected_stdout); + errors += self.compare_output("stderr", normalized_stderr.as_bytes(), &expected_stderr); + + if errors > 0 { + println!("To update references, run this command from build directory:"); + let relative_path_to_file = + self.testpaths.relative_dir + .join(self.testpaths.file.file_name().unwrap()); + println!("{}/update-references.sh '{}' '{}'", + self.config.src_base.display(), + self.config.build_base.display(), + relative_path_to_file.display()); + self.fatal(&format!("{} errors occurred comparing output.", errors)); + } + } + + fn normalize_output(&self, output: &str) -> String { + let parent_dir = self.testpaths.file.parent().unwrap(); + let parent_dir_str = parent_dir.display().to_string(); + output.replace(&parent_dir_str, "$DIR") + .replace("\\", "/") // windows, you know. + } + + fn expected_output_path(&self, kind: &str) -> PathBuf { + let extension = match self.revision { + Some(r) => format!("{}.{}", r, kind), + None => kind.to_string(), + }; + self.testpaths.file.with_extension(extension) + } + + fn load_expected_output(&self, path: &Path) -> Vec { + if !path.exists() { + return vec![]; + } + + let mut result = Vec::new(); + match File::open(path).and_then(|mut f| f.read_to_end(&mut result)) { + Ok(_) => result, + Err(e) => { + self.fatal(&format!("failed to load expected output from `{}`: {}", path.display(), e)) + } + } + } + + fn compare_output(&self, kind: &str, actual: &[u8], expected: &[u8]) -> usize { + if self.config.verbose { + println!("normalized {}:\n{}\n", kind, str::from_utf8(actual).unwrap_or("not utf8")); + println!("expected {}:\n{}\n", kind, str::from_utf8(expected).unwrap_or("not utf8")); + } + if actual == expected { + return 0; + } + + let output_file = self.output_base_name().with_extension(kind); + match File::create(&output_file).and_then(|mut f| f.write_all(actual)) { + Ok(()) => { } + Err(e) => { + self.fatal(&format!("failed to write {} to `{}`: {}", + kind, output_file.display(), e)) + } + } + + println!("\nThe actual {0} differed from the expected {0}.", kind); + println!("Actual {} saved to {}", kind, output_file.display()); + 1 + } } struct ProcArgs {