Auto merge of #33588 - nikomatsakis:compiletest-ui, r=acrichto

add UI testing framework

This adds a framework for capturing and tracking the precise output of rustc, which allows us to check all manner of minor details with the output. It's pretty strict right now -- the output must match almost exactly -- and hence maybe a bit too strict. But I figure we can add wildcards or whatever later. There is also a script intended to make updating the references easy, though the script could make things a *bit* easier (in particular, it'd be nice if it would find the build directory for you automatically).

One thing I was wondering about is the best way to test colors. Since windows doesn't embed those in the output stream, this test framework can't test colors on windows -- so I figure we can just write tests that are ignored on windows and which pass `--color=always` or whatever to rustc.

cc @jonathandturner
r? @alexcrichton

mk/tests.mk

@@ -274,6 +274,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-gdb-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-lldb-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-incremental-exec \
+	check-stage$(1)-T-$(2)-H-$(3)-ui-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-doc-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-pretty-exec
 
@@ -452,6 +453,9 @@ CODEGEN_CC := $(call rwildcard,$(S)src/test/codegen/,*.cc)
 CODEGEN_UNITS_RS := $(call rwildcard,$(S)src/test/codegen-units/,*.rs)
 INCREMENTAL_RS := $(call rwildcard,$(S)src/test/incremental/,*.rs)
 RMAKE_RS := $(wildcard $(S)src/test/run-make/*/Makefile)
+UI_RS := $(call rwildcard,$(S)src/test/ui/,*.rs) \
+         $(call rwildcard,$(S)src/test/ui/,*.stdout) \
+         $(call rwildcard,$(S)src/test/ui/,*.stderr)
 RUSTDOCCK_RS := $(call rwildcard,$(S)src/test/rustdoc/,*.rs)
 
 RPASS_TESTS := $(RPASS_RS)
@@ -469,6 +473,7 @@ CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
 CODEGEN_UNITS_TESTS := $(CODEGEN_UNITS_RS)
 INCREMENTAL_TESTS := $(INCREMENTAL_RS)
 RMAKE_TESTS := $(RMAKE_RS)
+UI_TESTS := $(UI_RS)
 RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)
 
 CTEST_SRC_BASE_rpass = run-pass
@@ -541,6 +546,11 @@ CTEST_BUILD_BASE_rmake = run-make
 CTEST_MODE_rmake = run-make
 CTEST_RUNTOOL_rmake = $(CTEST_RUNTOOL)
 
+CTEST_SRC_BASE_ui = ui
+CTEST_BUILD_BASE_ui = ui
+CTEST_MODE_ui = ui
+CTEST_RUNTOOL_ui = $(CTEST_RUNTOOL)
+
 CTEST_SRC_BASE_rustdocck = rustdoc
 CTEST_BUILD_BASE_rustdocck = rustdoc
 CTEST_MODE_rustdocck = rustdoc
@@ -672,7 +682,7 @@ CTEST_DEPS_codegen-units_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_UNITS_TESTS)
 CTEST_DEPS_incremental_$(1)-T-$(2)-H-$(3) = $$(INCREMENTAL_TESTS)
 CTEST_DEPS_rmake_$(1)-T-$(2)-H-$(3) = $$(RMAKE_TESTS) \
 	$$(CSREQ$(1)_T_$(3)_H_$(3)) $$(SREQ$(1)_T_$(2)_H_$(3))
-
+CTEST_DEPS_ui_$(1)-T-$(2)-H-$(3) = $$(UI_TESTS)
 CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
 		$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
 		$(S)src/etc/htmldocck.py
@@ -744,7 +754,7 @@ endef
 
 CTEST_NAMES = rpass rpass-valgrind rpass-full rfail-full cfail-full rfail cfail pfail \
 	debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck incremental \
-	rmake
+	rmake ui
 
 $(foreach host,$(CFG_HOST), \
  $(eval $(foreach target,$(CFG_TARGET), \
@@ -943,6 +953,7 @@ TEST_GROUPS = \
 	codegen \
 	codegen-units \
 	incremental \
+	ui \
 	doc \
 	$(foreach docname,$(DOC_NAMES),doc-$(docname)) \
 	pretty \

src/bootstrap/build/mod.rs

@@ -342,6 +342,14 @@ impl Build {
                     check::compiletest(self, &compiler, target.target,
                                        "codegen-units", "codegen-units");
                 }
+                CheckIncremental { compiler } => {
+                    check::compiletest(self, &compiler, target.target,
+                                       "incremental", "incremental");
+                }
+                CheckUi { compiler } => {
+                    check::compiletest(self, &compiler, target.target,
+                                       "ui", "ui");
+                }
                 CheckDebuginfo { compiler } => {
                     if target.target.contains("msvc") ||
                        target.target.contains("android") {

src/bootstrap/build/step.rs

@@ -111,6 +111,8 @@ macro_rules! targets {
             (check_pfail, CheckPFail { compiler: Compiler<'a> }),
             (check_codegen, CheckCodegen { compiler: Compiler<'a> }),
             (check_codegen_units, CheckCodegenUnits { compiler: Compiler<'a> }),
+            (check_incremental, CheckIncremental { compiler: Compiler<'a> }),
+            (check_ui, CheckUi { compiler: Compiler<'a> }),
             (check_debuginfo, CheckDebuginfo { compiler: Compiler<'a> }),
             (check_rustdoc, CheckRustdoc { compiler: Compiler<'a> }),
             (check_pretty, CheckPretty { compiler: Compiler<'a> }),
@@ -379,6 +381,8 @@ impl<'a> Step<'a> {
                     self.check_cfail(compiler),
                     self.check_rfail(compiler),
                     self.check_pfail(compiler),
+                    self.check_incremental(compiler),
+                    self.check_ui(compiler),
                     self.check_crate_std(compiler),
                     self.check_crate_test(compiler),
                     self.check_crate_rustc(compiler),
@@ -412,6 +416,8 @@ impl<'a> Step<'a> {
             Source::CheckPFail { compiler } |
             Source::CheckCodegen { compiler } |
             Source::CheckCodegenUnits { compiler } |
+            Source::CheckIncremental { compiler } |
+            Source::CheckUi { compiler } |
             Source::CheckRustdoc { compiler } |
             Source::CheckPretty { compiler } |
             Source::CheckCFail { compiler } |

src/test/ui/README.md

@@ -0,0 +1,31 @@
+# Guide to the UI Tests
+
+The UI tests are intended to capture the compiler's complete output,
+so that we can test all aspects of the presentation. They work by
+compiling a file (e.g., `hello_world/main.rs`), capturing the output,
+and then applying some normalization (see below). This normalized
+result is then compared against reference files named
+`hello_world/main.stderr` and `hello_world/main.stdout`. If either of
+those files doesn't exist, the output must be empty. If the test run
+fails, we will print out the current output, but it is also saved in
+`build/<target-triple>/test/ui/hello_world/main.stdout` (this path is
+printed as part of the test failure mesage), so you can run `diff` and
+so forth.
+
+# Editing and updating the reference files
+
+If you have changed the compiler's output intentionally, or you are
+making a new test, you can use the script `update-references.sh` to
+update the references. When you run the test framework, it will report
+various errors: in those errors is a command you can use to run the
+`update-references.sh` script, which will then copy over the files
+from the build directory and use them as the new reference. You can
+also just run `update-all-references.sh`. In both cases, you can run
+the script with `--help` to get a help message.
+
+# Normalization
+
+The normalization applied is aimed at filenames:
+
+- the test directory is replaced with `$DIR`
+- all backslashes (\) are converted to forward slashes (/) (for windows)

src/test/ui/hello_world/main.rs

@@ -0,0 +1,15 @@
+// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Test that compiling hello world succeeds with no output of any kind.
+
+fn main() {
+    println!("Hello, world!");
+}

src/test/ui/mismatched_types/main.rs

@@ -0,0 +1,17 @@
+// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// rustc-env:RUST_NEW_ERROR_FORMAT
+
+fn main() {
+    let x: u32 = (
+    );
+}
+

src/test/ui/mismatched_types/main.stderr

@@ -0,0 +1,8 @@
+error: mismatched types [--explain E0308]
+  --> $DIR/main.rs:14:18
+14 |>     let x: u32 = (
+   |>                  ^ expected u32, found ()
+note: expected type `u32`
+note:    found type `()`
+
+error: aborting due to previous error

src/test/ui/update-all-references.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# A script to update the references for all tests. The idea is that
+# you do a run, which will generate files in the build directory
+# containing the (normalized) actual output of the compiler. You then
+# run this script, which will copy those files over. If you find
+# yourself manually editing a foo.stderr file, you're doing it wrong.
+#
+# See all `update-references.sh`, if you just want to update a single test.
+
+if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" != "" ]]; then
+    echo "usage: $0 <build-directory>"
+    echo ""
+    echo "For example:"
+    echo "   $0 ../../../build/x86_64-apple-darwin/test/ui"
+fi
+
+BUILD_DIR=$PWD/$1
+MY_DIR=$(dirname $0)
+cd $MY_DIR
+find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR

src/test/ui/update-references.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# A script to update the references for particular tests. The idea is
+# that you do a run, which will generate files in the build directory
+# containing the (normalized) actual output of the compiler. This
+# script will then copy that output and replace the "expected output"
+# files. You can then commit the changes.
+#
+# If you find yourself manually editing a foo.stderr file, you're
+# doing it wrong.
+
+if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
+    echo "usage: $0 <build-directory> <relative-path-to-rs-files>"
+    echo ""
+    echo "For example:"
+    echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
+fi
+
+MYDIR=$(dirname $0)
+
+BUILD_DIR="$1"
+shift
+
+while [[ "$1" != "" ]]; do
+    STDERR_NAME="${1/%.rs/.stderr}"
+    STDOUT_NAME="${1/%.rs/.stdout}"
+    shift
+    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
+           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME > /dev/null); then
+        echo updating $MYDIR/$STDOUT_NAME
+        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    fi
+    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
+           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME > /dev/null); then
+        echo updating $MYDIR/$STDERR_NAME
+        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    fi
+done
+
+

src/tools/compiletest/src/common.rs

@@ -28,6 +28,7 @@ pub enum Mode {
     CodegenUnits,
     Incremental,
     RunMake,
+    Ui,
 }
 
 impl FromStr for Mode {
@@ -47,6 +48,7 @@ impl FromStr for Mode {
           "codegen-units" => Ok(CodegenUnits),
           "incremental" => Ok(Incremental),
           "run-make" => Ok(RunMake),
+          "ui" => Ok(Ui),
           _ => Err(()),
         }
     }
@@ -68,6 +70,7 @@ impl fmt::Display for Mode {
             CodegenUnits => "codegen-units",
             Incremental => "incremental",
             RunMake => "run-make",
+            Ui => "ui",
         }, f)
     }
 }

src/tools/compiletest/src/main.rs

@@ -50,6 +50,7 @@ pub mod runtest;
 pub mod common;
 pub mod errors;
 mod raise_fd_limit;
+mod uidiff;
 
 fn main() {
     #[cfg(cargobuild)]

src/tools/compiletest/src/runtest.rs

@@ -11,13 +11,14 @@
 use common::Config;
 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
 use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
-use common::{Incremental, RunMake};
+use common::{Incremental, RunMake, Ui};
 use errors::{self, ErrorKind, Error};
 use json;
 use header::TestProps;
 use header;
 use procsrv;
 use test::TestPaths;
+use uidiff;
 use util::logv;
 
 use std::env;
@@ -29,6 +30,7 @@ use std::io::prelude::*;
 use std::net::TcpStream;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Output, ExitStatus};
+use std::str;
 
 pub fn run(config: Config, testpaths: &TestPaths) {
     match &*config.target {
@@ -118,6 +120,7 @@ impl<'test> TestCx<'test> {
             CodegenUnits => self.run_codegen_units_test(),
             Incremental => self.run_incremental_test(),
             RunMake => self.run_rmake_test(),
+            Ui => self.run_ui_test(),
         }
     }
 
@@ -1314,6 +1317,7 @@ actual:\n\
             Codegen |
             Rustdoc |
             RunMake |
+            Ui |
             CodegenUnits => {
                 // do not use JSON output
             }
@@ -2096,6 +2100,96 @@ actual:\n\
         }
         fs::remove_dir(path)
     }
+
+    fn run_ui_test(&self) {
+        println!("ui: {}", self.testpaths.file.display());
+
+        let proc_res = self.compile_test();
+
+        let expected_stderr_path = self.expected_output_path("stderr");
+        let expected_stderr = self.load_expected_output(&expected_stderr_path);
+
+        let expected_stdout_path = self.expected_output_path("stdout");
+        let expected_stdout = self.load_expected_output(&expected_stdout_path);
+
+        let normalized_stdout = self.normalize_output(&proc_res.stdout);
+        let normalized_stderr = self.normalize_output(&proc_res.stderr);
+
+        let mut errors = 0;
+        errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
+        errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
+
+        if errors > 0 {
+            println!("To update references, run this command from build directory:");
+            let relative_path_to_file =
+                self.testpaths.relative_dir
+                              .join(self.testpaths.file.file_name().unwrap());
+            println!("{}/update-references.sh '{}' '{}'",
+                     self.config.src_base.display(),
+                     self.config.build_base.display(),
+                     relative_path_to_file.display());
+            self.fatal_proc_rec(&format!("{} errors occurred comparing output.", errors),
+                                &proc_res);
+        }
+    }
+
+    fn normalize_output(&self, output: &str) -> String {
+        let parent_dir = self.testpaths.file.parent().unwrap();
+        let parent_dir_str = parent_dir.display().to_string();
+        output.replace(&parent_dir_str, "$DIR")
+              .replace("\\", "/") // normalize for paths on windows
+              .replace("\r\n", "\n") // normalize for linebreaks on windows
+              .replace("\t", "\\t") // makes tabs visible
+    }
+
+    fn expected_output_path(&self, kind: &str) -> PathBuf {
+        let extension = match self.revision {
+            Some(r) => format!("{}.{}", r, kind),
+            None => kind.to_string(),
+        };
+        self.testpaths.file.with_extension(extension)
+    }
+
+    fn load_expected_output(&self, path: &Path) -> String {
+        if !path.exists() {
+            return String::new();
+        }
+
+        let mut result = String::new();
+        match File::open(path).and_then(|mut f| f.read_to_string(&mut result)) {
+            Ok(_) => result,
+            Err(e) => {
+                self.fatal(&format!("failed to load expected output from `{}`: {}",
+                                    path.display(), e))
+            }
+        }
+    }
+
+    fn compare_output(&self, kind: &str, actual: &str, expected: &str) -> usize {
+        if actual == expected {
+            return 0;
+        }
+
+        println!("normalized {}:\n{}\n", kind, actual);
+        println!("expected {}:\n{}\n", kind, expected);
+        println!("diff of {}:\n", kind);
+        for line in uidiff::diff_lines(actual, expected) {
+            println!("{}", line);
+        }
+
+        let output_file = self.output_base_name().with_extension(kind);
+        match File::create(&output_file).and_then(|mut f| f.write_all(actual.as_bytes())) {
+            Ok(()) => { }
+            Err(e) => {
+                self.fatal(&format!("failed to write {} to `{}`: {}",
+                                    kind, output_file.display(), e))
+            }
+        }
+
+        println!("\nThe actual {0} differed from the expected {0}.", kind);
+        println!("Actual {} saved to {}", kind, output_file.display());
+        1
+    }
 }
 
 struct ProcArgs {

src/tools/compiletest/src/uidiff.rs

@@ -0,0 +1,76 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Code for checking whether the output of the compiler matches what is
+//! expected.
+
+pub fn diff_lines(actual: &str, expected: &str) -> Vec<String> {
+    // mega simplistic diff algorithm that just prints the things added/removed
+    zip_all(actual.lines(), expected.lines()).enumerate().filter_map(|(i, (a,e))| {
+        match (a, e) {
+            (Some(a), Some(e)) => {
+                if lines_match(e, a) {
+                    None
+                } else {
+                    Some(format!("{:3} - |{}|\n    + |{}|\n", i, e, a))
+                }
+            },
+            (Some(a), None) => {
+                Some(format!("{:3} -\n    + |{}|\n", i, a))
+            },
+            (None, Some(e)) => {
+                Some(format!("{:3} - |{}|\n    +\n", i, e))
+            },
+            (None, None) => panic!("Cannot get here")
+        }
+    }).collect()
+}
+
+fn lines_match(expected: &str, mut actual: &str) -> bool {
+    for (i, part) in expected.split("[..]").enumerate() {
+        match actual.find(part) {
+            Some(j) => {
+                if i == 0 && j != 0 {
+                    return false
+                }
+                actual = &actual[j + part.len()..];
+            }
+            None => {
+                return false
+            }
+        }
+    }
+    actual.is_empty() || expected.ends_with("[..]")
+}
+
+struct ZipAll<I1: Iterator, I2: Iterator> {
+    first: I1,
+    second: I2,
+}
+
+impl<T, I1: Iterator<Item=T>, I2: Iterator<Item=T>> Iterator for ZipAll<I1, I2> {
+    type Item = (Option<T>, Option<T>);
+    fn next(&mut self) -> Option<(Option<T>, Option<T>)> {
+        let first = self.first.next();
+        let second = self.second.next();
+
+        match (first, second) {
+            (None, None) => None,
+            (a, b) => Some((a, b))
+        }
+    }
+}
+
+fn zip_all<T, I1: Iterator<Item=T>, I2: Iterator<Item=T>>(a: I1, b: I2) -> ZipAll<I1, I2> {
+    ZipAll {
+        first: a,
+        second: b,
+    }
+}