From 02b8533ac8c1cc4521b1da91e52c1327f656c44a Mon Sep 17 00:00:00 2001 From: Philipp Hansch Date: Sun, 24 Mar 2019 14:06:47 +0100 Subject: [PATCH] Add a way to track Rustfix UI test coverage This came out of the first Rustfix WG meeting. One of the goals is to enable Rustfix tests for all UI tests that trigger lints with `MachineApplicable` suggestions. In order to do that we first want to create a tracking issue that lists all files with missing `// run-rustfix` headers. This PR adds a `--rustfix-coverage` flag to `./x.py` and compiletest to list the files with the missing headers in `/tmp/rustfix_missing_coverage.txt`. From that file we can create the tracking issue and at some point also enforce the `// run-rustfix` flag on UI tests with `MachineApplicable` lints. --- src/bootstrap/flags.rs | 15 +++++++++++++++ src/bootstrap/test.rs | 4 ++++ src/tools/compiletest/src/common.rs | 5 +++++ src/tools/compiletest/src/main.rs | 19 +++++++++++++++++++ src/tools/compiletest/src/runtest.rs | 24 +++++++++++++++++++++++- 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 0f9a4271ac0..23719378c84 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -56,6 +56,7 @@ pub enum Subcommand { rustc_args: Vec, fail_fast: bool, doc_tests: DocTests, + rustfix_coverage: bool, }, Bench { paths: Vec, @@ -188,6 +189,12 @@ To learn more about a subcommand, run `./x.py -h`" "mode describing what file the actual ui output will be compared to", "COMPARE MODE", ); + opts.optflag( + "", + "rustfix-coverage", + "enable this to generate a Rustfix coverage file, which is saved in \ + `/tmp/rustfix_missing_coverage.txt`", + ); } "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); @@ -363,6 +370,7 @@ Arguments: test_args: matches.opt_strs("test-args"), rustc_args: matches.opt_strs("rustc-args"), fail_fast: !matches.opt_present("no-fail-fast"), + rustfix_coverage: matches.opt_present("rustfix-coverage"), doc_tests: if matches.opt_present("doc") { DocTests::Only } else if matches.opt_present("no-doc") { @@ -467,6 +475,13 @@ impl Subcommand { } } + pub fn rustfix_coverage(&self) -> bool { + match *self { + Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, + _ => false, + } + } + pub fn compare_mode(&self) -> Option<&str> { match *self { Subcommand::Test { diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index bbe1872d395..41c73f307b6 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1284,6 +1284,10 @@ impl Step for Compiletest { cmd.arg("--android-cross-path").arg(""); } + if builder.config.cmd.rustfix_coverage() { + cmd.arg("--rustfix-coverage"); + } + builder.ci_env.force_coloring_in_ci(&mut cmd); let _folder = builder.fold_output(|| format!("test_{}", suite)); diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 80b8a8b728b..fe0c4f14940 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -245,6 +245,11 @@ pub struct Config { /// mode describing what file the actual ui output will be compared to pub compare_mode: Option, + /// If true, this will generate a coverage file with UI test files that run `MachineApplicable` + /// lints but are missing `run-rustfix` annotations. The generated coverage file is created in + /// `/tmp/rustfix_missing_coverage.txt` + pub rustfix_coverage: bool, + // Configuration for various run-make tests frobbing things like C compilers // or querying about various LLVM component information. pub cc: String, diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 86cdadade10..1045ea1bf0c 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -233,6 +233,12 @@ pub fn parse_config(args: Vec) -> Config { "mode describing what file the actual ui output will be compared to", "COMPARE MODE", ) + .optflag( + "", + "rustfix-coverage", + "enable this to generate a Rustfix coverage file, which is saved in \ + `/tmp/rustfix_missing_coverage.txt`", + ) .optflag("h", "help", "show this message"); let (argv0, args_) = args.split_first().unwrap(); @@ -336,6 +342,7 @@ pub fn parse_config(args: Vec) -> Config { color, remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from), compare_mode: matches.opt_str("compare-mode").map(CompareMode::parse), + rustfix_coverage: matches.opt_present("rustfix-coverage"), cc: matches.opt_str("cc").unwrap(), cxx: matches.opt_str("cxx").unwrap(), @@ -475,6 +482,18 @@ pub fn run_tests(config: &Config) { let _ = fs::remove_dir_all("tmp/partitioning-tests"); } + // If we want to collect rustfix coverage information, + // we first make sure that the coverage file does not exist. + // It will be created later on. + if config.rustfix_coverage { + let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt"); + if coverage_file_path.exists() { + if let Err(e) = fs::remove_file(coverage_file_path) { + panic!("Could not delete {} due to {}", coverage_file_path.display(), e) + } + } + } + let opts = test_opts(config); let tests = make_tests(config); // sadly osx needs some file descriptor limits raised for running tests in diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 3e3499edf60..0ca656bb133 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -19,7 +19,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::env; use std::ffi::{OsStr, OsString}; use std::fmt; -use std::fs::{self, create_dir_all, File}; +use std::fs::{self, create_dir_all, File, OpenOptions}; use std::hash::{Hash, Hasher}; use std::io::prelude::*; use std::io::{self, BufReader}; @@ -2818,6 +2818,28 @@ impl<'test> TestCx<'test> { if self.config.compare_mode.is_some() { // don't test rustfix with nll right now + } else if self.config.rustfix_coverage { + // Find out which tests have `MachineApplicable` suggestions but are missing + // `run-rustfix` or `run-rustfix-only-machine-applicable` headers + let suggestions = get_suggestions_from_json( + &proc_res.stderr, + &HashSet::new(), + Filter::MachineApplicableOnly + ).unwrap(); + if suggestions.len() > 0 + && !self.props.run_rustfix + && !self.props.rustfix_only_machine_applicable { + let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt"); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(coverage_file_path) + .expect("could not create or open file"); + + if let Err(_) = writeln!(file, "{}", self.testpaths.file.display()) { + panic!("couldn't write to {}", coverage_file_path.display()); + } + } } else if self.props.run_rustfix { // Apply suggestions from rustc to the code itself let unfixed_code = self