From 678e2177dcc089bb5355819d11ee22d7703126d4 Mon Sep 17 00:00:00 2001 From: jyn Date: Sun, 9 Jul 2023 21:42:23 -0500 Subject: [PATCH 1/2] bootstrap: print test name on failure even with `verbose-tests=false` This makes it much easier to know which test failed without having to wait for compiletest to completely finish running. Before: ``` Testing stage0 compiletest suite=ui mode=ui (x86_64-unknown-linux-gnu) running 15274 tests iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 88/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 176/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 264/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 352/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 440/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 528/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiFFiiiiiii ... ``` After: ``` Testing stage0 compiletest suite=ui mode=ui (x86_64-unknown-linux-gnu) running 15274 tests iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 88/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 176/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 264/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 352/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 440/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii 528/15274 iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii [ui] tests/ui/associated-type-bounds/implied-in-supertrait.rs ... F [ui] tests/ui/associated-type-bounds/return-type-notation/basic.rs#next_with ... F iiiiiiiiiiiii ... ``` This serves a similar use case to the existing RUSTC_TEST_FAIL_FAST, but is on by default and as a result much more discoverable. We should consider unifying RUSTC_TEST_FAIL_FAST with the `--no-fail-fast` flag in the future for consistency and discoverability. --- src/bootstrap/src/utils/render_tests.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/utils/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs index bfbb53f8c8121..cbd01606a895a 100644 --- a/src/bootstrap/src/utils/render_tests.rs +++ b/src/bootstrap/src/utils/render_tests.rs @@ -166,7 +166,7 @@ impl<'a> Renderer<'a> { println!(); } - fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, _: &TestOutcome) { + fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { if let Some(total) = self.tests_count { let total = total.to_string(); @@ -178,7 +178,7 @@ impl<'a> Renderer<'a> { } self.terse_tests_in_line += 1; - self.builder.colored_stdout(|stdout| outcome.write_short(stdout)).unwrap(); + self.builder.colored_stdout(|stdout| outcome.write_short(stdout, &test.name)).unwrap(); let _ = std::io::stdout().flush(); } @@ -300,7 +300,7 @@ enum Outcome<'a> { } impl Outcome<'_> { - fn write_short(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + fn write_short(&self, writer: &mut dyn WriteColor, name: &str) -> Result<(), std::io::Error> { match self { Outcome::Ok => { writer.set_color(ColorSpec::new().set_fg(Some(Color::Green)))?; @@ -311,8 +311,11 @@ impl Outcome<'_> { write!(writer, "b")?; } Outcome::Failed => { + // Put failed tests on their own line and include the test name, so that it's faster + // to see which test failed without having to wait for them all to run. + writeln!(writer)?; writer.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?; - write!(writer, "F")?; + writeln!(writer, "{name} ... F")?; } Outcome::Ignored { .. } => { writer.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?; From 8bfe9dbae20628bfcea390556c40a991984692ec Mon Sep 17 00:00:00 2001 From: jyn Date: Sat, 4 Nov 2023 21:40:38 -0400 Subject: [PATCH 2/2] libtest: Print the names of failed tests eagerly Previously, libtest would wait until all tests finished running to print the progress, which made it annoying to run many tests at once (since you don't know which have failed). Change it to print the names as soon as they fail. This also adds a test for the terse output; previously it was untested. --- library/test/src/formatters/terse.rs | 35 ++++++-- tests/ui/test-attrs/terse.rs | 125 +++++++++++++++++++++++++++ tests/ui/test-attrs/terse.run.stdout | 31 +++++++ 3 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 tests/ui/test-attrs/terse.rs create mode 100644 tests/ui/test-attrs/terse.run.stdout diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs index 2931ca6ead0ac..22c28e4954eff 100644 --- a/library/test/src/formatters/terse.rs +++ b/library/test/src/formatters/terse.rs @@ -23,6 +23,7 @@ pub(crate) struct TerseFormatter { max_name_len: usize, test_count: usize, + test_column: usize, total_test_count: usize, } @@ -39,6 +40,7 @@ impl TerseFormatter { max_name_len, is_multithreaded, test_count: 0, + test_column: 0, total_test_count: 0, // initialized later, when write_run_start is called } } @@ -47,8 +49,20 @@ impl TerseFormatter { self.write_short_result(".", term::color::GREEN) } - pub fn write_failed(&mut self) -> io::Result<()> { - self.write_short_result("F", term::color::RED) + pub fn write_failed(&mut self, name: &str) -> io::Result<()> { + // Put failed tests on their own line and include the test name, so that it's faster + // to see which test failed without having to wait for them all to run. + + // normally, we write the progress unconditionally, even if the previous line was cut short. + // but if this is the very first column, no short results will have been printed and we'll end up with *only* the progress on the line. + // avoid this. + if self.test_column != 0 { + self.write_progress()?; + } + self.test_count += 1; + self.write_plain(format!("{name} --- "))?; + self.write_pretty("FAILED", term::color::RED)?; + self.write_plain("\n") } pub fn write_ignored(&mut self) -> io::Result<()> { @@ -65,15 +79,22 @@ impl TerseFormatter { color: term::color::Color, ) -> io::Result<()> { self.write_pretty(result, color)?; - if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + self.test_count += 1; + self.test_column += 1; + if self.test_column % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { // We insert a new line regularly in order to flush the // screen when dealing with line-buffered output (e.g., piping to // `stamp` in the rust CI). - let out = format!(" {}/{}\n", self.test_count + 1, self.total_test_count); - self.write_plain(out)?; + self.write_progress()?; } - self.test_count += 1; + Ok(()) + } + + fn write_progress(&mut self) -> io::Result<()> { + let out = format!(" {}/{}\n", self.test_count, self.total_test_count); + self.write_plain(out)?; + self.test_column = 0; Ok(()) } @@ -213,7 +234,7 @@ impl OutputFormatter for TerseFormatter { match *result { TestResult::TrOk => self.write_ok(), TestResult::TrFailed | TestResult::TrFailedMsg(_) | TestResult::TrTimedFail => { - self.write_failed() + self.write_failed(desc.name.as_slice()) } TestResult::TrIgnored => self.write_ignored(), TestResult::TrBench(ref bs) => { diff --git a/tests/ui/test-attrs/terse.rs b/tests/ui/test-attrs/terse.rs new file mode 100644 index 0000000000000..ab9d5cc19bd6c --- /dev/null +++ b/tests/ui/test-attrs/terse.rs @@ -0,0 +1,125 @@ +//@ compile-flags: --test +//@ run-fail +//@ run-flags: --test-threads=1 --quiet +//@ check-run-results +//@ exec-env:RUST_BACKTRACE=0 +//@ normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" +//@ ignore-emscripten no threads support +//@ needs-unwind + +#[test] +fn abc() { + panic!(); +} + +#[test] +fn foo() { + panic!(); +} + +#[test] +fn foo2() { + panic!(); +} + +// run a whole bunch of tests so we can see what happens when we go over 88 columns +#[test] fn f0() {} +#[test] fn f1() {} +#[test] fn f2() {} +#[test] fn f3() {} +#[test] fn f4() {} +#[test] fn f5() {} +#[test] fn f6() {} +#[test] fn f7() {} +#[test] fn f8() {} +#[test] fn f9() {} +#[test] fn f10() {} +#[test] fn f11() {} +#[test] fn f12() {} +#[test] fn f13() {} +#[test] fn f14() {} +#[test] fn f15() {} +#[test] fn f16() {} +#[test] fn f17() {} +#[test] fn f18() {} +#[test] fn f19() {} +#[test] fn f20() {} +#[test] fn f21() {} +#[test] fn f22() {} +#[test] fn f23() {} +#[test] fn f24() {} +#[test] fn f25() {} +#[test] fn f26() {} +#[test] fn f27() {} +#[test] fn f28() {} +#[test] fn f29() {} +#[test] fn f30() {} +#[test] fn f31() {} +#[test] fn f32() {} +#[test] fn f33() {} +#[test] fn f34() {} +#[test] fn f35() {} +#[test] fn f36() {} +#[test] fn f37() {} +#[test] fn f38() {} +#[test] fn f39() {} +#[test] fn f40() {} +#[test] fn f41() {} +#[test] fn f42() {} +#[test] fn f43() {} +#[test] fn f44() {} +#[test] fn f45() {} +#[test] fn f46() {} +#[test] fn f47() {} +#[test] fn f48() {} +#[test] fn f49() {} +#[test] fn f50() {} +#[test] fn f51() {} +#[test] fn f52() {} +#[test] fn f53() {} +#[test] fn f54() {} +#[test] fn f55() {} +#[test] fn f56() {} +#[test] fn f57() {} +#[test] fn f58() {} +#[test] fn f59() {} +#[test] fn f60() {} +#[test] fn f61() {} +#[test] fn f62() {} +#[test] fn f63() {} +#[test] fn f64() {} +#[test] fn f65() {} +#[test] fn f66() {} +#[test] fn f67() {} +#[test] fn f68() {} +#[test] fn f69() {} +#[test] fn f70() {} +#[test] fn f71() {} +#[test] fn f72() {} +#[test] fn f73() {} +#[test] fn f74() {} +#[test] fn f75() {} +#[test] fn f76() {} +#[test] fn f77() {} +#[test] fn f78() {} +#[test] fn f79() {} +#[test] fn f80() {} +#[test] fn f81() {} +#[test] fn f82() {} +#[test] fn f83() {} +#[test] fn f84() {} +#[test] fn f85() {} +#[test] fn f86() {} +#[test] fn f87() {} +#[test] fn f88() {} +#[test] fn f89() {} +#[test] fn f90() {} +#[test] fn f91() {} +#[test] fn f92() {} +#[test] fn f93() {} +#[test] fn f94() {} +#[test] fn f95() {} +#[test] fn f96() {} +#[test] fn f97() {} +#[test] fn f98() {} +#[test] fn f99() {} diff --git a/tests/ui/test-attrs/terse.run.stdout b/tests/ui/test-attrs/terse.run.stdout new file mode 100644 index 0000000000000..2b361361ae888 --- /dev/null +++ b/tests/ui/test-attrs/terse.run.stdout @@ -0,0 +1,31 @@ + +running 103 tests +abc --- FAILED +....................................................................................... 88/103 +............. 101/103 +foo --- FAILED +foo2 --- FAILED + +failures: + +---- abc stdout ---- +thread 'abc' panicked at $DIR/terse.rs:12:5: +explicit panic +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- foo stdout ---- +thread 'foo' panicked at $DIR/terse.rs:17:5: +explicit panic + +---- foo2 stdout ---- +thread 'foo2' panicked at $DIR/terse.rs:22:5: +explicit panic + + +failures: + abc + foo + foo2 + +test result: FAILED. 100 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME +