From 60dd83ea85853f6a31f8998eb80ce47446fdb785 Mon Sep 17 00:00:00 2001 From: Paul Woolcock Date: Wed, 24 May 2017 13:58:37 -0400 Subject: [PATCH] add `allow_fail` test attribute This change allows the user to add an `#[allow_fail]` attribute to tests that will cause the test to compile & run, but if the test fails it will not cause the entire test run to fail. The test output will show the failure, but in yellow instead of red, and also indicate that it was an allowed failure. --- src/librustdoc/html/markdown.rs | 41 +++++++++++++---------- src/librustdoc/test.rs | 3 +- src/libsyntax/feature_gate.rs | 1 + src/libsyntax/test.rs | 14 ++++++-- src/libtest/lib.rs | 23 +++++++++++-- src/test/run-pass/test-allow-fail-attr.rs | 23 +++++++++++++ src/tools/compiletest/src/main.rs | 1 + 7 files changed, 81 insertions(+), 25 deletions(-) create mode 100644 src/test/run-pass/test-allow-fail-attr.rs diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index bea13397eca..5a56c33b806 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -769,7 +769,7 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position block_info.should_panic, block_info.no_run, block_info.ignore, block_info.test_harness, block_info.compile_fail, block_info.error_codes, - line, filename); + line, filename, block_info.allow_fail); } else { tests.add_old_test(text, filename); } @@ -859,7 +859,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Sp block_info.should_panic, block_info.no_run, block_info.ignore, block_info.test_harness, block_info.compile_fail, block_info.error_codes, - line, filename); + line, filename, block_info.allow_fail); prev_offset = offset; } Event::Start(Tag::Header(level)) => { @@ -889,6 +889,7 @@ struct LangString { test_harness: bool, compile_fail: bool, error_codes: Vec, + allow_fail: bool, } impl LangString { @@ -902,6 +903,7 @@ impl LangString { test_harness: false, compile_fail: false, error_codes: Vec::new(), + allow_fail: false, } } @@ -930,6 +932,7 @@ impl LangString { } "no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; } "ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; } + "allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; } "rust" => { data.rust = true; seen_rust_tags = true; } "test_harness" => { data.test_harness = true; @@ -1118,7 +1121,7 @@ mod tests { fn test_lang_string_parse() { fn t(s: &str, should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool, - compile_fail: bool, error_codes: Vec) { + compile_fail: bool, allow_fail: bool, error_codes: Vec) { assert_eq!(LangString::parse(s), LangString { should_panic: should_panic, no_run: no_run, @@ -1128,25 +1131,27 @@ mod tests { compile_fail: compile_fail, error_codes: error_codes, original: s.to_owned(), + allow_fail: allow_fail, }) } // marker | should_panic| no_run| ignore| rust | test_harness| compile_fail - // | error_codes - t("", false, false, false, true, false, false, Vec::new()); - t("rust", false, false, false, true, false, false, Vec::new()); - t("sh", false, false, false, false, false, false, Vec::new()); - t("ignore", false, false, true, true, false, false, Vec::new()); - t("should_panic", true, false, false, true, false, false, Vec::new()); - t("no_run", false, true, false, true, false, false, Vec::new()); - t("test_harness", false, false, false, true, true, false, Vec::new()); - t("compile_fail", false, true, false, true, false, true, Vec::new()); - t("{.no_run .example}", false, true, false, true, false, false, Vec::new()); - t("{.sh .should_panic}", true, false, false, false, false, false, Vec::new()); - t("{.example .rust}", false, false, false, true, false, false, Vec::new()); - t("{.test_harness .rust}", false, false, false, true, true, false, Vec::new()); - t("text, no_run", false, true, false, false, false, false, Vec::new()); - t("text,no_run", false, true, false, false, false, false, Vec::new()); + // | allow_fail | error_codes + t("", false, false, false, true, false, false, false, Vec::new()); + t("rust", false, false, false, true, false, false, false, Vec::new()); + t("sh", false, false, false, false, false, false, false, Vec::new()); + t("ignore", false, false, true, true, false, false, false, Vec::new()); + t("should_panic", true, false, false, true, false, false, false, Vec::new()); + t("no_run", false, true, false, true, false, false, false, Vec::new()); + t("test_harness", false, false, false, true, true, false, false, Vec::new()); + t("compile_fail", false, true, false, true, false, true, false, Vec::new()); + t("allow_fail", false, false, false, true, false, false, true, Vec::new()); + t("{.no_run .example}", false, true, false, true, false, false, false, Vec::new()); + t("{.sh .should_panic}", true, false, false, false, false, false, false, Vec::new()); + t("{.example .rust}", false, false, false, true, false, false, false, Vec::new()); + t("{.test_harness .rust}", false, false, false, true, true, false, false, Vec::new()); + t("text, no_run", false, true, false, false, false, false, false, Vec::new()); + t("text,no_run", false, true, false, false, false, false, false, Vec::new()); } #[test] diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index cfe2fad0fa4..4766778eed1 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -467,7 +467,7 @@ impl Collector { pub fn add_test(&mut self, test: String, should_panic: bool, no_run: bool, should_ignore: bool, as_test_harness: bool, compile_fail: bool, error_codes: Vec, - line: usize, filename: String) { + line: usize, filename: String, allow_fail: bool) { let name = self.generate_name(line, &filename); // to be removed when hoedown is removed if self.render_type == RenderType::Pulldown { @@ -499,6 +499,7 @@ impl Collector { ignore: should_ignore, // compiler failures are test failures should_panic: testing::ShouldPanic::No, + allow_fail: allow_fail, }, testfn: testing::DynTestFn(box move |()| { let panic = io::set_panic(None); diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index d7d3a70f3c7..07db5b83331 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -534,6 +534,7 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG ("derive", Normal, Ungated), ("should_panic", Normal, Ungated), ("ignore", Normal, Ungated), + ("allow_fail", Normal, Ungated), ("no_implicit_prelude", Normal, Ungated), ("reexport_test_harness_main", Normal, Ungated), ("link_args", Normal, Ungated), diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs index a0d1785c6ff..86f5f42eac7 100644 --- a/src/libsyntax/test.rs +++ b/src/libsyntax/test.rs @@ -52,7 +52,8 @@ struct Test { path: Vec , bench: bool, ignore: bool, - should_panic: ShouldPanic + should_panic: ShouldPanic, + allow_fail: bool, } struct TestCtxt<'a> { @@ -133,7 +134,8 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> { path: self.cx.path.clone(), bench: is_bench_fn(&self.cx, &i), ignore: is_ignored(&i), - should_panic: should_panic(&i, &self.cx) + should_panic: should_panic(&i, &self.cx), + allow_fail: is_allowed_fail(&i), }; self.cx.testfns.push(test); self.tests.push(i.ident); @@ -383,6 +385,10 @@ fn is_ignored(i: &ast::Item) -> bool { i.attrs.iter().any(|attr| attr.check_name("ignore")) } +fn is_allowed_fail(i: &ast::Item) -> bool { + i.attrs.iter().any(|attr| attr.check_name("allow_fail")) +} + fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic { match i.attrs.iter().find(|attr| attr.check_name("should_panic")) { Some(attr) => { @@ -668,6 +674,7 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P { } } }; + let allow_fail_expr = ecx.expr_bool(span, test.allow_fail); // self::test::TestDesc { ... } let desc_expr = ecx.expr_struct( @@ -675,7 +682,8 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P { test_path("TestDesc"), vec![field("name", name_expr), field("ignore", ignore_expr), - field("should_panic", fail_expr)]); + field("should_panic", fail_expr), + field("allow_fail", allow_fail_expr)]); let mut visible_path = match cx.toplevel_reexport { diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 2094fd8898d..c68039f21ec 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -212,6 +212,7 @@ pub struct TestDesc { pub name: TestName, pub ignore: bool, pub should_panic: ShouldPanic, + pub allow_fail: bool, } #[derive(Clone)] @@ -523,6 +524,7 @@ pub enum TestResult { TrFailed, TrFailedMsg(String), TrIgnored, + TrAllowedFail, TrMetrics(MetricMap), TrBench(BenchSamples), } @@ -543,6 +545,7 @@ struct ConsoleTestState { passed: usize, failed: usize, ignored: usize, + allowed_fail: usize, filtered_out: usize, measured: usize, metrics: MetricMap, @@ -572,6 +575,7 @@ impl ConsoleTestState { passed: 0, failed: 0, ignored: 0, + allowed_fail: 0, filtered_out: 0, measured: 0, metrics: MetricMap::new(), @@ -594,6 +598,10 @@ impl ConsoleTestState { self.write_short_result("ignored", "i", term::color::YELLOW) } + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW) + } + pub fn write_metric(&mut self) -> io::Result<()> { self.write_pretty("metric", term::color::CYAN) } @@ -669,6 +677,7 @@ impl ConsoleTestState { TrOk => self.write_ok(), TrFailed | TrFailedMsg(_) => self.write_failed(), TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), TrMetrics(ref mm) => { self.write_metric()?; self.write_plain(&format!(": {}\n", mm.fmt_metrics())) @@ -702,6 +711,7 @@ impl ConsoleTestState { TrFailed => "failed".to_owned(), TrFailedMsg(ref msg) => format!("failed: {}", msg), TrIgnored => "ignored".to_owned(), + TrAllowedFail => "failed (allowed)".to_owned(), TrMetrics(ref mm) => mm.fmt_metrics(), TrBench(ref bs) => fmt_bench_samples(bs), }, @@ -761,7 +771,7 @@ impl ConsoleTestState { } pub fn write_run_finish(&mut self) -> io::Result { - assert!(self.passed + self.failed + self.ignored + self.measured == self.total); + assert!(self.passed + self.failed + self.ignored + self.measured + self.allowed_fail == self.total); if self.options.display_output { self.write_outputs()?; @@ -778,9 +788,10 @@ impl ConsoleTestState { } else { self.write_pretty("FAILED", term::color::RED)?; } - let s = format!(". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + let s = format!(". {} passed; {} failed; {} allowed to fail; {} ignored; {} measured; {} filtered out\n\n", self.passed, self.failed, + self.allowed_fail, self.ignored, self.measured, self.filtered_out); @@ -891,6 +902,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu st.not_failures.push((test, stdout)); } TrIgnored => st.ignored += 1, + TrAllowedFail => st.allowed_fail += 1, TrMetrics(mm) => { let tname = test.name; let MetricMap(mm) = mm; @@ -1471,8 +1483,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box>) -> Tes .unwrap_or(false) { TrOk } else { - TrFailedMsg(format!("Panic did not include expected string '{}'", msg)) + if desc.allow_fail { + TrAllowedFail + } else { + TrFailedMsg(format!("Panic did not include expected string '{}'", msg)) + } }, + _ if desc.allow_fail => TrAllowedFail, _ => TrFailed, } } diff --git a/src/test/run-pass/test-allow-fail-attr.rs b/src/test/run-pass/test-allow-fail-attr.rs new file mode 100644 index 00000000000..7d750d51dcd --- /dev/null +++ b/src/test/run-pass/test-allow-fail-attr.rs @@ -0,0 +1,23 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: --test + +#[test] +#[allow_fail] +fn test1() { + panic!(); +} + +#[test] +#[allow_fail] +fn test2() { + assert!(true); +} diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index c88ffba357a..b4663b0ee6c 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -476,6 +476,7 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn name: make_test_name(config, testpaths), ignore: ignore, should_panic: should_panic, + allow_fail: false, }, testfn: make_test_closure(config, testpaths), }