diff --git a/src/libtest/formatters/json.rs b/src/libtest/formatters/json.rs new file mode 100644 index 0000000000000..d323d50f702ba --- /dev/null +++ b/src/libtest/formatters/json.rs @@ -0,0 +1,229 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) struct JsonFormatter { + out: OutputLocation, +} + +impl JsonFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { out } + } + + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref())?; + self.out.write_all(b"\n") + } + + fn write_event( + &mut self, + ty: &str, + name: &str, + evt: &str, + extra: Option, + ) -> io::Result<()> { + if let Some(extras) = extra { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + ty, + name, + evt, + extras + )) + } else { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + ty, + name, + evt + )) + } + } +} + +impl OutputFormatter for JsonFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, + test_count + )) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + desc.name + )) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()> { + match *result { + TrOk => self.write_event("test", desc.name.as_slice(), "ok", None), + + TrFailed => { + let extra_data = if stdout.len() > 0 { + Some(format!( + r#""stdout": "{}""#, + EscapedString(String::from_utf8_lossy(stdout)) + )) + } else { + None + }; + + self.write_event("test", desc.name.as_slice(), "failed", extra_data) + } + + TrFailedMsg(ref m) => { + self.write_event( + "test", + desc.name.as_slice(), + "failed", + Some(format!(r#""message": "{}""#, EscapedString(m))), + ) + } + + TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None), + + TrAllowedFail => { + self.write_event("test", desc.name.as_slice(), "allowed_failure", None) + } + + TrBench(ref bs) => { + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + let mbps = if bs.mb_s == 0 { + "".into() + } else { + format!(r#", "mib_per_second": {}"#, bs.mb_s) + }; + + let line = format!( + "{{ \"type\": \"bench\", \ + \"name\": \"{}\", \ + \"median\": {}, \ + \"deviation\": {}{} }}", + desc.name, + median, + deviation, + mbps + ); + + self.write_message(&*line) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + desc.name + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + + self.write_message(&*format!( + "{{ \"type\": \"suite\", \ + \"event\": \"{}\", \ + \"passed\": {}, \ + \"failed\": {}, \ + \"allowed_fail\": {}, \ + \"ignored\": {}, \ + \"measured\": {}, \ + \"filtered_out\": \"{}\" }}", + if state.failed == 0 { "ok" } else { "failed" }, + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ))?; + + Ok(state.failed == 0) + } +} + +/// A formatting utility used to print strings with characters in need of escaping. +/// Base code taken form `libserialize::json::escape_str` +struct EscapedString>(S); + +impl> ::std::fmt::Display for EscapedString { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + let mut start = 0; + + for (i, byte) in self.0.as_ref().bytes().enumerate() { + let escaped = match byte { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\x00' => "\\u0000", + b'\x01' => "\\u0001", + b'\x02' => "\\u0002", + b'\x03' => "\\u0003", + b'\x04' => "\\u0004", + b'\x05' => "\\u0005", + b'\x06' => "\\u0006", + b'\x07' => "\\u0007", + b'\x08' => "\\b", + b'\t' => "\\t", + b'\n' => "\\n", + b'\x0b' => "\\u000b", + b'\x0c' => "\\f", + b'\r' => "\\r", + b'\x0e' => "\\u000e", + b'\x0f' => "\\u000f", + b'\x10' => "\\u0010", + b'\x11' => "\\u0011", + b'\x12' => "\\u0012", + b'\x13' => "\\u0013", + b'\x14' => "\\u0014", + b'\x15' => "\\u0015", + b'\x16' => "\\u0016", + b'\x17' => "\\u0017", + b'\x18' => "\\u0018", + b'\x19' => "\\u0019", + b'\x1a' => "\\u001a", + b'\x1b' => "\\u001b", + b'\x1c' => "\\u001c", + b'\x1d' => "\\u001d", + b'\x1e' => "\\u001e", + b'\x1f' => "\\u001f", + b'\x7f' => "\\u007f", + _ => { + continue; + } + }; + + if start < i { + f.write_str(&self.0.as_ref()[start..i])?; + } + + f.write_str(escaped)?; + + start = i + 1; + } + + if start != self.0.as_ref().len() { + f.write_str(&self.0.as_ref()[start..])?; + } + + Ok(()) + } +} diff --git a/src/libtest/formatters/mod.rs b/src/libtest/formatters/mod.rs new file mode 100644 index 0000000000000..24c7929076c1d --- /dev/null +++ b/src/libtest/formatters/mod.rs @@ -0,0 +1,32 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +mod pretty; +mod json; +mod terse; + +pub(crate) use self::pretty::PrettyFormatter; +pub(crate) use self::json::JsonFormatter; +pub(crate) use self::terse::TerseFormatter; + +pub(crate) trait OutputFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()>; + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; +} diff --git a/src/libtest/formatters/pretty.rs b/src/libtest/formatters/pretty.rs new file mode 100644 index 0000000000000..f2064deefce62 --- /dev/null +++ b/src/libtest/formatters/pretty.rs @@ -0,0 +1,247 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) struct PrettyFormatter { + out: OutputLocation, + use_color: bool, + + /// Number of columns to fill when aligning names + max_name_len: usize, + + is_multithreaded: bool, +} + +impl PrettyFormatter { + pub fn new( + out: OutputLocation, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + PrettyFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + } + } + + #[cfg(test)] + pub fn output_location(&self) -> &OutputLocation { + &self.out + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result("ok", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("FAILED", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("ignored", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("FAILED (allowed)", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + self.write_plain("\n") + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_successes(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + + Ok(()) + } +} + +impl OutputFormatter for PrettyFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // When running tests concurrently, we should not print + // the test's name as the result will be mis-aligned. + // When running the tests serially, we print the name here so + // that the user can see which test hangs. + if !self.is_multithreaded { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, + TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_successes(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed, + state.ignored, + state.measured, + state.filtered_out + ) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/formatters/terse.rs b/src/libtest/formatters/terse.rs new file mode 100644 index 0000000000000..88689485144c0 --- /dev/null +++ b/src/libtest/formatters/terse.rs @@ -0,0 +1,246 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) struct TerseFormatter { + out: OutputLocation, + use_color: bool, + is_multithreaded: bool, + /// Number of columns to fill when aligning names + max_name_len: usize, + + test_count: usize, +} + +impl TerseFormatter { + pub fn new( + out: OutputLocation, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + TerseFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + test_count: 0, + } + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result(".", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("F", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("i", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("a", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + // we insert a new line every 100 dots in order to flush the + // screen when dealing with line-buffered output (e.g. piping to + // `stamp` in the rust CI). + self.write_plain("\n")?; + } + + self.test_count += 1; + Ok(()) + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + + Ok(()) + } +} + +impl OutputFormatter for TerseFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // Remnants from old libtest code that used the padding value + // in order to indicate benchmarks. + // When running benchmarks, terse-mode should still print their name as if + // it is the Pretty formatter. + if !self.is_multithreaded && desc.name.padding() == PadOnRight { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, + TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_outputs(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed, + state.ignored, + state.measured, + state.filtered_out + ) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index f7880d3c4d854..ffa27688cf1a7 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // @@ -71,6 +71,7 @@ use std::sync::mpsc::{channel, Sender}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Instant, Duration}; +use std::borrow::Cow; const TEST_WARN_TIMEOUT_S: u64 = 60; const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode @@ -84,6 +85,9 @@ pub mod test { } pub mod stats; +mod formatters; + +use formatters::{OutputFormatter, PrettyFormatter, TerseFormatter, JsonFormatter}; // The name of a test. By convention this follows the rules for rust // paths; i.e. it should be a series of identifiers separated by double @@ -94,14 +98,33 @@ pub mod stats; pub enum TestName { StaticTestName(&'static str), DynTestName(String), + AlignedTestName(Cow<'static, str>, NamePadding), } impl TestName { fn as_slice(&self) -> &str { match *self { StaticTestName(s) => s, DynTestName(ref s) => s, + AlignedTestName(ref s, _) => &*s, } } + + fn padding(&self) -> NamePadding { + match self { + &AlignedTestName(_, p) => p, + _ => PadNone, + } + } + + fn with_padding(&self, padding: NamePadding) -> TestName { + let name = match self { + &TestName::StaticTestName(name) => Cow::Borrowed(name), + &TestName::DynTestName(ref name) => Cow::Owned(name.clone()), + &TestName::AlignedTestName(ref name, _) => name.clone(), + }; + + TestName::AlignedTestName(name, padding) + } } impl fmt::Display for TestName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -109,7 +132,7 @@ impl fmt::Display for TestName { } } -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum NamePadding { PadNone, PadOnRight, @@ -217,10 +240,7 @@ pub struct Metric { impl Metric { pub fn new(value: f64, noise: f64) -> Metric { - Metric { - value, - noise, - } + Metric { value, noise } } } @@ -232,9 +252,7 @@ pub struct Options { impl Options { pub fn new() -> Options { - Options { - display_output: false, - } + Options { display_output: false } } pub fn display_output(mut self, display_output: bool) -> Options { @@ -274,25 +292,24 @@ pub fn test_main(args: &[String], tests: Vec, options: Options) { // rather than a &[]. pub fn test_main_static(tests: &[TestDescAndFn]) { let args = env::args().collect::>(); - let owned_tests = tests.iter() - .map(|t| { - match t.testfn { - StaticTestFn(f) => { - TestDescAndFn { - testfn: StaticTestFn(f), - desc: t.desc.clone(), - } - } - StaticBenchFn(f) => { - TestDescAndFn { - testfn: StaticBenchFn(f), - desc: t.desc.clone(), - } - } - _ => panic!("non-static tests passed to test::test_main_static"), - } - }) - .collect(); + let owned_tests = tests + .iter() + .map(|t| match t.testfn { + StaticTestFn(f) => { + TestDescAndFn { + testfn: StaticTestFn(f), + desc: t.desc.clone(), + } + } + StaticBenchFn(f) => { + TestDescAndFn { + testfn: StaticBenchFn(f), + desc: t.desc.clone(), + } + } + _ => panic!("non-static tests passed to test::test_main_static"), + }) + .collect(); test_main(&args, owned_tests, Options::new()) } @@ -303,6 +320,13 @@ pub enum ColorConfig { NeverColor, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum OutputFormat { + Pretty, + Terse, + Json, +} + #[derive(Debug)] pub struct TestOpts { pub list: bool, @@ -314,7 +338,7 @@ pub struct TestOpts { pub logfile: Option, pub nocapture: bool, pub color: ColorConfig, - pub quiet: bool, + pub format: OutputFormat, pub test_threads: Option, pub skip: Vec, pub options: Options, @@ -333,7 +357,7 @@ impl TestOpts { logfile: None, nocapture: false, color: AutoColor, - quiet: false, + format: OutputFormat::Pretty, test_threads: None, skip: vec![], options: Options::new(), @@ -351,26 +375,76 @@ fn optgroups() -> getopts::Options { .optflag("", "bench", "Run benchmarks instead of tests") .optflag("", "list", "List all tests and benchmarks") .optflag("h", "help", "Display this message (longer with --help)") - .optopt("", "logfile", "Write logs to the specified file instead \ - of stdout", "PATH") - .optflag("", "nocapture", "don't capture stdout/stderr of each \ - task, allow printing directly") - .optopt("", "test-threads", "Number of threads used for running tests \ - in parallel", "n_threads") - .optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \ - be used multiple times)","FILTER") - .optflag("q", "quiet", "Display one character per test instead of one line") - .optflag("", "exact", "Exactly match filters rather than by substring") - .optopt("", "color", "Configure coloring of output: + .optopt( + "", + "logfile", + "Write logs to the specified file instead \ + of stdout", + "PATH", + ) + .optflag( + "", + "nocapture", + "don't capture stdout/stderr of each \ + task, allow printing directly", + ) + .optopt( + "", + "test-threads", + "Number of threads used for running tests \ + in parallel", + "n_threads", + ) + .optmulti( + "", + "skip", + "Skip tests whose names contain FILTER (this flag can \ + be used multiple times)", + "FILTER", + ) + .optflag( + "q", + "quiet", + "Display one character per test instead of one line. \ + Alias to --format=terse", + ) + .optflag( + "", + "exact", + "Exactly match filters rather than by substring", + ) + .optopt( + "", + "color", + "Configure coloring of output: auto = colorize if stdout is a tty and tests are run on serially (default); always = always colorize output; - never = never colorize output;", "auto|always|never"); - return opts + never = never colorize output;", + "auto|always|never", + ) + .optopt( + "", + "format", + "Configure formatting of output: + pretty = Print verbose output; + terse = Display one character per test; + json = Output a json document", + "pretty|terse|json", + ) + .optopt( + "Z", + "", + "Enable nightly-only flags: + unstable-options = Allow use of experimental features", + "unstable-options", + ); + return opts; } fn usage(binary: &str, options: &getopts::Options) { let message = format!("Usage: {} [OPTIONS] [FILTER]", binary); - println!(r#"{usage} + println!( + r#"{usage} The FILTER string is tested against the name of all tests, and only those tests whose names contain the filter are run. @@ -397,11 +471,23 @@ Test Attributes: test, then the test runner will ignore these tests during normal test runs. Running with --ignored will run these tests."#, - usage = options.usage(&message)); + usage = options.usage(&message) + ); +} + +// FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566 +fn is_nightly() -> bool { + // Whether this is a feature-staged build, i.e. on the beta or stable channel + let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some(); + // Whether we should enable unstable features for bootstrapping + let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok(); + + bootstrap || !disable_unstable_features } // Parses command line arguments into test options pub fn parse_opts(args: &[String]) -> Option { + let mut allow_unstable = false; let opts = optgroups(); let args = args.get(1..).unwrap_or(args); let matches = match opts.parse(args) { @@ -409,6 +495,24 @@ pub fn parse_opts(args: &[String]) -> Option { Err(f) => return Some(Err(f.to_string())), }; + if let Some(opt) = matches.opt_str("Z") { + if !is_nightly() { + return Some(Err( + "the option `Z` is only accepted on the nightly compiler" + .into(), + )); + } + + match &*opt { + "unstable-options" => { + allow_unstable = true; + } + _ => { + return Some(Err("Unrecognized option to `Z`".into())); + } + } + }; + if matches.opt_present("h") { usage(&args[0], &opts); return None; @@ -435,22 +539,25 @@ pub fn parse_opts(args: &[String]) -> Option { if !nocapture { nocapture = match env::var("RUST_TEST_NOCAPTURE") { Ok(val) => &val != "0", - Err(_) => false + Err(_) => false, }; } let test_threads = match matches.opt_str("test-threads") { - Some(n_str) => + Some(n_str) => { match n_str.parse::() { - Ok(0) => - return Some(Err(format!("argument for --test-threads must not be 0"))), + Ok(0) => return Some(Err(format!("argument for --test-threads must not be 0"))), Ok(n) => Some(n), - Err(e) => - return Some(Err(format!("argument for --test-threads must be a number > 0 \ - (error: {})", e))) - }, - None => - None, + Err(e) => { + return Some(Err(format!( + "argument for --test-threads must be a number > 0 \ + (error: {})", + e + ))) + } + } + } + None => None, }; let color = match matches.opt_str("color").as_ref().map(|s| &**s) { @@ -459,9 +566,34 @@ pub fn parse_opts(args: &[String]) -> Option { Some("never") => NeverColor, Some(v) => { - return Some(Err(format!("argument for --color must be auto, always, or never (was \ + return Some(Err(format!( + "argument for --color must be auto, always, or never (was \ + {})", + v + ))) + } + }; + + let format = match matches.opt_str("format").as_ref().map(|s| &**s) { + None if quiet => OutputFormat::Terse, + Some("pretty") | None => OutputFormat::Pretty, + Some("terse") => OutputFormat::Terse, + Some("json") => { + if !allow_unstable { + return Some(Err( + "The \"json\" format is only accepted on the nightly compiler" + .into(), + )); + } + OutputFormat::Json + } + + Some(v) => { + return Some(Err(format!( + "argument for --format must be pretty, terse, or json (was \ {})", - v))) + v + ))) } }; @@ -475,7 +607,7 @@ pub fn parse_opts(args: &[String]) -> Option { logfile, nocapture, color, - quiet, + format, test_threads, skip: matches.opt_strs("skip"), options: Options::new(), @@ -507,11 +639,24 @@ enum OutputLocation { Raw(T), } -struct ConsoleTestState { +impl Write for OutputLocation { + fn write(&mut self, buf: &[u8]) -> io::Result { + match *self { + Pretty(ref mut term) => term.write(buf), + Raw(ref mut stdout) => stdout.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match *self { + Pretty(ref mut term) => term.flush(), + Raw(ref mut stdout) => stdout.flush(), + } + } +} + +struct ConsoleTestState { log_out: Option, - out: OutputLocation, - use_color: bool, - quiet: bool, total: usize, passed: usize, failed: usize, @@ -522,26 +667,18 @@ struct ConsoleTestState { metrics: MetricMap, failures: Vec<(TestDesc, Vec)>, not_failures: Vec<(TestDesc, Vec)>, - max_name_len: usize, // number of columns to fill when aligning names options: Options, } -impl ConsoleTestState { - pub fn new(opts: &TestOpts, _: Option) -> io::Result> { +impl ConsoleTestState { + pub fn new(opts: &TestOpts) -> io::Result { let log_out = match opts.logfile { Some(ref path) => Some(File::create(path)?), None => None, }; - let out = match term::stdout() { - None => Raw(io::stdout()), - Some(t) => Pretty(t), - }; Ok(ConsoleTestState { - out, log_out, - use_color: use_color(opts), - quiet: opts.quiet, total: 0, passed: 0, failed: 0, @@ -552,119 +689,10 @@ impl ConsoleTestState { metrics: MetricMap::new(), failures: Vec::new(), not_failures: Vec::new(), - max_name_len: 0, options: opts.options, }) } - pub fn write_ok(&mut self) -> io::Result<()> { - self.write_short_result("ok", ".", term::color::GREEN) - } - - pub fn write_failed(&mut self) -> io::Result<()> { - self.write_short_result("FAILED", "F", term::color::RED) - } - - pub fn write_ignored(&mut self) -> io::Result<()> { - self.write_short_result("ignored", "i", term::color::YELLOW) - } - - pub fn write_allowed_fail(&mut self) -> io::Result<()> { - self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW) - } - - pub fn write_bench(&mut self) -> io::Result<()> { - self.write_pretty("bench", term::color::CYAN) - } - - pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color) - -> io::Result<()> { - if self.quiet { - self.write_pretty(quiet, color)?; - if self.current_test_count() % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { - // we insert a new line every 100 dots in order to flush the - // screen when dealing with line-buffered output (e.g. piping to - // `stamp` in the rust CI). - self.write_plain("\n")?; - } - Ok(()) - } else { - self.write_pretty(verbose, color)?; - self.write_plain("\n") - } - } - - pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { - match self.out { - Pretty(ref mut term) => { - if self.use_color { - term.fg(color)?; - } - term.write_all(word.as_bytes())?; - if self.use_color { - term.reset()?; - } - term.flush() - } - Raw(ref mut stdout) => { - stdout.write_all(word.as_bytes())?; - stdout.flush() - } - } - } - - pub fn write_plain>(&mut self, s: S) -> io::Result<()> { - let s = s.as_ref(); - match self.out { - Pretty(ref mut term) => { - term.write_all(s.as_bytes())?; - term.flush() - } - Raw(ref mut stdout) => { - stdout.write_all(s.as_bytes())?; - stdout.flush() - } - } - } - - pub fn write_run_start(&mut self, len: usize) -> io::Result<()> { - self.total = len; - let noun = if len != 1 { - "tests" - } else { - "test" - }; - self.write_plain(&format!("\nrunning {} {}\n", len, noun)) - } - - pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> { - if self.quiet && align != PadOnRight { - Ok(()) - } else { - let name = test.padded_name(self.max_name_len, align); - self.write_plain(&format!("test {} ... ", name)) - } - } - - pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> { - match *result { - TrOk => self.write_ok(), - TrFailed | TrFailedMsg(_) => self.write_failed(), - TrIgnored => self.write_ignored(), - TrAllowedFail => self.write_allowed_fail(), - TrBench(ref bs) => { - self.write_bench()?; - self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) - } - } - } - - pub fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_plain(&format!("test {} has been running for over {} seconds\n", - desc.name, - TEST_WARN_TIMEOUT_S)) - } - pub fn write_log>(&mut self, msg: S) -> io::Result<()> { let msg = msg.as_ref(); match self.log_out { @@ -674,114 +702,23 @@ impl ConsoleTestState { } pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> { - self.write_log( - format!("{} {}\n", - match *result { - TrOk => "ok".to_owned(), - TrFailed => "failed".to_owned(), - TrFailedMsg(ref msg) => format!("failed: {}", msg), - TrIgnored => "ignored".to_owned(), - TrAllowedFail => "failed (allowed)".to_owned(), - TrBench(ref bs) => fmt_bench_samples(bs), - }, - test.name)) - } - - pub fn write_failures(&mut self) -> io::Result<()> { - self.write_plain("\nfailures:\n")?; - let mut failures = Vec::new(); - let mut fail_out = String::new(); - for &(ref f, ref stdout) in &self.failures { - failures.push(f.name.to_string()); - if !stdout.is_empty() { - fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); - let output = String::from_utf8_lossy(stdout); - fail_out.push_str(&output); - fail_out.push_str("\n"); - } - } - if !fail_out.is_empty() { - self.write_plain("\n")?; - self.write_plain(&fail_out)?; - } - - self.write_plain("\nfailures:\n")?; - failures.sort(); - for name in &failures { - self.write_plain(&format!(" {}\n", name))?; - } - Ok(()) - } - - pub fn write_outputs(&mut self) -> io::Result<()> { - self.write_plain("\nsuccesses:\n")?; - let mut successes = Vec::new(); - let mut stdouts = String::new(); - for &(ref f, ref stdout) in &self.not_failures { - successes.push(f.name.to_string()); - if !stdout.is_empty() { - stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); - let output = String::from_utf8_lossy(stdout); - stdouts.push_str(&output); - stdouts.push_str("\n"); - } - } - if !stdouts.is_empty() { - self.write_plain("\n")?; - self.write_plain(&stdouts)?; - } - - self.write_plain("\nsuccesses:\n")?; - successes.sort(); - for name in &successes { - self.write_plain(&format!(" {}\n", name))?; - } - Ok(()) + self.write_log(format!( + "{} {}\n", + match *result { + TrOk => "ok".to_owned(), + TrFailed => "failed".to_owned(), + TrFailedMsg(ref msg) => format!("failed: {}", msg), + TrIgnored => "ignored".to_owned(), + TrAllowedFail => "failed (allowed)".to_owned(), + TrBench(ref bs) => fmt_bench_samples(bs), + }, + test.name + )) } fn current_test_count(&self) -> usize { self.passed + self.failed + self.ignored + self.measured + self.allowed_fail } - - pub fn write_run_finish(&mut self) -> io::Result { - assert!(self.current_test_count() == self.total); - - if self.options.display_output { - self.write_outputs()?; - } - let success = self.failed == 0; - if !success { - self.write_failures()?; - } - - self.write_plain("\ntest result: ")?; - if success { - // There's no parallelism at this point so it's safe to use color - self.write_pretty("ok", term::color::GREEN)?; - } else { - self.write_pretty("FAILED", term::color::RED)?; - } - let s = if self.allowed_fail > 0 { - format!( - ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", - self.passed, - self.failed + self.allowed_fail, - self.allowed_fail, - self.ignored, - self.measured, - self.filtered_out) - } else { - format!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", - self.passed, - self.failed, - self.ignored, - self.measured, - self.filtered_out) - }; - self.write_plain(&s)?; - return Ok(success); - } } // Format a number with thousands separators @@ -815,19 +752,30 @@ pub fn fmt_bench_samples(bs: &BenchSamples) -> String { let median = bs.ns_iter_summ.median as usize; let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; - output.write_fmt(format_args!("{:>11} ns/iter (+/- {})", - fmt_thousands_sep(median, ','), - fmt_thousands_sep(deviation, ','))) - .unwrap(); + output + .write_fmt(format_args!( + "{:>11} ns/iter (+/- {})", + fmt_thousands_sep(median, ','), + fmt_thousands_sep(deviation, ',') + )) + .unwrap(); if bs.mb_s != 0 { - output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap(); + output + .write_fmt(format_args!(" = {} MB/s", bs.mb_s)) + .unwrap(); } output } // List the tests to console, and optionally to logfile. Filters are honored. pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Result<()> { - let mut st = ConsoleTestState::new(opts, None::)?; + let mut output = match term::stdout() { + None => Raw(io::stdout()), + Some(t) => Pretty(t), + }; + + let quiet = opts.format == OutputFormat::Terse; + let mut st = ConsoleTestState::new(opts)?; let mut ntest = 0; let mut nbench = 0; @@ -835,14 +783,24 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res for test in filter_tests(&opts, tests) { use TestFn::*; - let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test; + let TestDescAndFn { + desc: TestDesc { name, .. }, + testfn, + } = test; let fntype = match testfn { - StaticTestFn(..) | DynTestFn(..) => { ntest += 1; "test" }, - StaticBenchFn(..) | DynBenchFn(..) => { nbench += 1; "benchmark" }, + StaticTestFn(..) | DynTestFn(..) => { + ntest += 1; + "test" + } + StaticBenchFn(..) | + DynBenchFn(..) => { + nbench += 1; + "benchmark" + } }; - st.write_plain(format!("{}: {}\n", name, fntype))?; + writeln!(output, "{}: {}", name, fntype)?; st.write_log(format!("{} {}\n", fntype, name))?; } @@ -853,13 +811,16 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res } } - if !opts.quiet { + if !quiet { if ntest != 0 || nbench != 0 { - st.write_plain("\n")?; + writeln!(output, "")?; } - st.write_plain(format!("{}, {}\n", + + writeln!(output, + "{}, {}", plural(ntest, "test"), - plural(nbench, "benchmark")))?; + plural(nbench, "benchmark") + )?; } Ok(()) @@ -867,16 +828,23 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res // A simple console test runner pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Result { + fn callback( + event: &TestEvent, + st: &mut ConsoleTestState, + out: &mut OutputFormatter, + ) -> io::Result<()> { - fn callback(event: &TestEvent, st: &mut ConsoleTestState) -> io::Result<()> { match (*event).clone() { - TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()), + TeFiltered(ref filtered_tests) => { + st.total = filtered_tests.len(); + out.write_run_start(filtered_tests.len()) + } TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), - TeWait(ref test, padding) => st.write_test_start(test, padding), - TeTimeout(ref test) => st.write_timeout(test), + TeWait(ref test) => out.write_test_start(test), + TeTimeout(ref test) => out.write_timeout(test), TeResult(test, result, stdout) => { st.write_log_result(&test, &result)?; - st.write_result(&result)?; + out.write_result(&test, &result, &*stdout)?; match result { TrOk => { st.passed += 1; @@ -885,9 +853,11 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu TrIgnored => st.ignored += 1, TrAllowedFail => st.allowed_fail += 1, TrBench(bs) => { - st.metrics.insert_metric(test.name.as_slice(), - bs.ns_iter_summ.median, - bs.ns_iter_summ.max - bs.ns_iter_summ.min); + st.metrics.insert_metric( + test.name.as_slice(), + bs.ns_iter_summ.median, + bs.ns_iter_summ.max - bs.ns_iter_summ.min, + ); st.measured += 1 } TrFailed => { @@ -897,9 +867,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu TrFailedMsg(msg) => { st.failed += 1; let mut stdout = stdout; - stdout.extend_from_slice( - format!("note: {}", msg).as_bytes() - ); + stdout.extend_from_slice(format!("note: {}", msg).as_bytes()); st.failures.push((test, stdout)); } } @@ -908,19 +876,47 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu } } - let mut st = ConsoleTestState::new(opts, None::)?; + let output = match term::stdout() { + None => Raw(io::stdout()), + Some(t) => Pretty(t), + }; + + let max_name_len = tests + .iter() + .max_by_key(|t| len_if_padded(*t)) + .map(|t| t.desc.name.as_slice().len()) + .unwrap_or(0); + + let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1; + + let mut out: Box = match opts.format { + OutputFormat::Pretty => Box::new(PrettyFormatter::new( + output, + use_color(opts), + max_name_len, + is_multithreaded, + )), + OutputFormat::Terse => Box::new(TerseFormatter::new( + output, + use_color(opts), + max_name_len, + is_multithreaded, + )), + OutputFormat::Json => Box::new(JsonFormatter::new(output)), + }; + let mut st = ConsoleTestState::new(opts)?; fn len_if_padded(t: &TestDescAndFn) -> usize { match t.testfn.padding() { PadNone => 0, PadOnRight => t.desc.name.as_slice().len(), } } - if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { - let n = t.desc.name.as_slice(); - st.max_name_len = n.len(); - } - run_tests(opts, tests, |x| callback(&x, &mut st))?; - return st.write_run_finish(); + + run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?; + + assert!(st.current_test_count() == st.total); + + return out.write_run_finish(&st); } #[test] @@ -939,11 +935,10 @@ fn should_sort_failures_before_printing_them() { allow_fail: false, }; - let mut st = ConsoleTestState { + let mut out = PrettyFormatter::new(Raw(Vec::new()), false, 10, false); + + let st = ConsoleTestState { log_out: None, - out: Raw(Vec::new()), - use_color: false, - quiet: false, total: 0, passed: 0, failed: 0, @@ -951,17 +946,16 @@ fn should_sort_failures_before_printing_them() { allowed_fail: 0, filtered_out: 0, measured: 0, - max_name_len: 10, metrics: MetricMap::new(), failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], options: Options::new(), not_failures: Vec::new(), }; - st.write_failures().unwrap(); - let s = match st.out { - Raw(ref m) => String::from_utf8_lossy(&m[..]), - Pretty(_) => unreachable!(), + out.write_failures(&st).unwrap(); + let s = match out.output_location() { + &Raw(ref m) => String::from_utf8_lossy(&m[..]), + &Pretty(_) => unreachable!(), }; let apos = s.find("a").unwrap(); @@ -1009,7 +1003,7 @@ fn stdout_isatty() -> bool { #[derive(Clone)] pub enum TestEvent { TeFiltered(Vec), - TeWait(TestDesc, NamePadding), + TeWait(TestDesc), TeResult(TestDesc, TestResult, Vec), TeTimeout(TestDesc), TeFilteredOut(usize), @@ -1017,9 +1011,19 @@ pub enum TestEvent { pub type MonitorMsg = (TestDesc, TestResult, Vec); +struct Sink(Arc>>); +impl Write for Sink { + fn write(&mut self, data: &[u8]) -> io::Result { + Write::write(&mut *self.0.lock().unwrap(), data) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) -> io::Result<()> - where F: FnMut(TestEvent) -> io::Result<()> +where + F: FnMut(TestEvent) -> io::Result<()>, { use std::collections::HashMap; use std::sync::mpsc::RecvTimeoutError; @@ -1031,27 +1035,29 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) filtered_tests = convert_benchmarks_to_tests(filtered_tests); } + let filtered_tests = { + let mut filtered_tests = filtered_tests; + for test in filtered_tests.iter_mut() { + test.desc.name = test.desc.name.with_padding(test.testfn.padding()); + } + + filtered_tests + }; + let filtered_out = tests_len - filtered_tests.len(); callback(TeFilteredOut(filtered_out))?; - let filtered_descs = filtered_tests.iter() - .map(|t| t.desc.clone()) - .collect(); + let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect(); callback(TeFiltered(filtered_descs))?; let (filtered_tests, filtered_benchs): (Vec<_>, _) = - filtered_tests.into_iter().partition(|e| { - match e.testfn { - StaticTestFn(_) | DynTestFn(_) => true, - _ => false, - } + filtered_tests.into_iter().partition(|e| match e.testfn { + StaticTestFn(_) | DynTestFn(_) => true, + _ => false, }); - let concurrency = match opts.test_threads { - Some(n) => n, - None => get_concurrency(), - }; + let concurrency = opts.test_threads.unwrap_or_else(get_concurrency); let mut remaining = filtered_tests; remaining.reverse(); @@ -1063,8 +1069,13 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) fn get_timed_out_tests(running_tests: &mut HashMap) -> Vec { let now = Instant::now(); - let timed_out = running_tests.iter() - .filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone())} else { None }) + let timed_out = running_tests + .iter() + .filter_map(|(desc, timeout)| if &now >= timeout { + Some(desc.clone()) + } else { + None + }) .collect(); for test in &timed_out { running_tests.remove(test); @@ -1079,13 +1090,14 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) *next_timeout - now } else { Duration::new(0, 0) - }}) + } + }) }; if concurrency == 1 { while !remaining.is_empty() { let test = remaining.pop().unwrap(); - callback(TeWait(test.desc.clone(), test.testfn.padding()))?; + callback(TeWait(test.desc.clone()))?; run_test(opts, !opts.run_tests, test, tx.clone()); let (test, result, stdout) = rx.recv().unwrap(); callback(TeResult(test, result, stdout))?; @@ -1096,6 +1108,7 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) let test = remaining.pop().unwrap(); let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); running_tests.insert(test.desc.clone(), timeout); + callback(TeWait(test.desc.clone()))?; //here no pad run_test(opts, !opts.run_tests, test, tx.clone()); pending += 1; } @@ -1119,7 +1132,6 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) let (desc, result, stdout) = res.unwrap(); running_tests.remove(&desc); - callback(TeWait(desc.clone(), PadNone))?; callback(TeResult(desc, result, stdout))?; pending -= 1; } @@ -1128,7 +1140,7 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) if opts.bench_benchmarks { // All benchmarks run at the end, in serial. for b in filtered_benchs { - callback(TeWait(b.desc.clone(), b.testfn.padding()))?; + callback(TeWait(b.desc.clone()))?; run_test(opts, false, b, tx.clone()); let (test, result, stdout) = rx.recv().unwrap(); callback(TeResult(test, result, stdout))?; @@ -1145,8 +1157,10 @@ fn get_concurrency() -> usize { match opt_n { Some(n) if n > 0 => n, _ => { - panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.", - s) + panic!( + "RUST_TEST_THREADS is `{}`, should be a positive integer.", + s + ) } } } @@ -1203,10 +1217,8 @@ fn get_concurrency() -> usize { unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize } } - #[cfg(any(target_os = "freebsd", - target_os = "dragonfly", - target_os = "bitrig", - target_os = "netbsd"))] + #[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "bitrig", + target_os = "netbsd"))] fn num_cpus() -> usize { use std::ptr; @@ -1219,12 +1231,14 @@ fn get_concurrency() -> usize { if cpus < 1 { let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; unsafe { - libc::sysctl(mib.as_mut_ptr(), - 2, - &mut cpus as *mut _ as *mut _, - &mut cpus_size as *mut _ as *mut _, - ptr::null_mut(), - 0); + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ); } if cpus < 1 { cpus = 1; @@ -1242,12 +1256,14 @@ fn get_concurrency() -> usize { let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; unsafe { - libc::sysctl(mib.as_mut_ptr(), - 2, - &mut cpus as *mut _ as *mut _, - &mut cpus_size as *mut _ as *mut _, - ptr::null_mut(), - 0); + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ); } if cpus < 1 { cpus = 1; @@ -1269,27 +1285,27 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec filtered, Some(ref filter) => { - filtered.into_iter() - .filter(|test| { - if opts.filter_exact { - test.desc.name.as_slice() == &filter[..] - } else { - test.desc.name.as_slice().contains(&filter[..]) - } - }) - .collect() + filtered + .into_iter() + .filter(|test| if opts.filter_exact { + test.desc.name.as_slice() == &filter[..] + } else { + test.desc.name.as_slice().contains(&filter[..]) + }) + .collect() } }; // Skip tests that match any of the skip filters - filtered = filtered.into_iter() - .filter(|t| !opts.skip.iter().any(|sf| { - if opts.filter_exact { - t.desc.name.as_slice() == &sf[..] - } else { - t.desc.name.as_slice().contains(&sf[..]) - } - })) + filtered = filtered + .into_iter() + .filter(|t| { + !opts.skip.iter().any(|sf| if opts.filter_exact { + t.desc.name.as_slice() == &sf[..] + } else { + t.desc.name.as_slice().contains(&sf[..]) + }) + }) .collect(); // Maybe pull out the ignored test and unignore them @@ -1298,9 +1314,12 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec Option { if test.desc.ignore { - let TestDescAndFn {desc, testfn} = test; + let TestDescAndFn { desc, testfn } = test; Some(TestDescAndFn { - desc: TestDesc { ignore: false, ..desc }, + desc: TestDesc { + ignore: false, + ..desc + }, testfn, }) } else { @@ -1311,7 +1330,9 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec) -> Vec f, - }; - TestDescAndFn { - desc: x.desc, - testfn, - } - }).collect() + f => f, + }; + TestDescAndFn { + desc: x.desc, + testfn, + } + }) + .collect() } -pub fn run_test(opts: &TestOpts, - force_ignore: bool, - test: TestDescAndFn, - monitor_ch: Sender) { +pub fn run_test( + opts: &TestOpts, + force_ignore: bool, + test: TestDescAndFn, + monitor_ch: Sender, +) { - let TestDescAndFn {desc, testfn} = test; + let TestDescAndFn { desc, testfn } = test; - let ignore_because_panic_abort = - cfg!(target_arch = "wasm32") && + let ignore_because_panic_abort = cfg!(target_arch = "wasm32") && !cfg!(target_os = "emscripten") && desc.should_panic != ShouldPanic::No; @@ -1364,16 +1387,6 @@ pub fn run_test(opts: &TestOpts, monitor_ch: Sender, nocapture: bool, testfn: Box) { - struct Sink(Arc>>); - impl Write for Sink { - fn write(&mut self, data: &[u8]) -> io::Result { - Write::write(&mut *self.0.lock().unwrap(), data) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - // Buffer for capturing standard I/O let data = Arc::new(Mutex::new(Vec::new())); let data2 = data.clone(); @@ -1383,7 +1396,7 @@ pub fn run_test(opts: &TestOpts, let oldio = if !nocapture { Some(( io::set_print(Some(Box::new(Sink(data2.clone())))), - io::set_panic(Some(Box::new(Sink(data2)))) + io::set_panic(Some(Box::new(Sink(data2)))), )) } else { None @@ -1398,21 +1411,18 @@ pub fn run_test(opts: &TestOpts, let test_result = calc_result(&desc, result); let stdout = data.lock().unwrap().to_vec(); - monitor_ch.send((desc.clone(), test_result, stdout)).unwrap(); + monitor_ch + .send((desc.clone(), test_result, stdout)) + .unwrap(); }; // If the platform is single-threaded we're just going to run // the test synchronously, regardless of the concurrency // level. - let supports_threads = - !cfg!(target_os = "emscripten") && - !cfg!(target_arch = "wasm32"); + let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32"); if supports_threads { - let cfg = thread::Builder::new().name(match name { - DynTestName(ref name) => name.clone(), - StaticTestName(name) => name.to_owned(), - }); + let cfg = thread::Builder::new().name(name.as_slice().to_owned()); cfg.spawn(runtest).unwrap(); } else { runtest(); @@ -1421,14 +1431,16 @@ pub fn run_test(opts: &TestOpts, match testfn { DynBenchFn(bencher) => { - let bs = ::bench::benchmark(|harness| bencher.run(harness)); - monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap(); - return; + ::bench::benchmark(desc, + monitor_ch, + opts.nocapture, + |harness| bencher.run(harness)); } StaticBenchFn(benchfn) => { - let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness)); - monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap(); - return; + ::bench::benchmark(desc, + monitor_ch, + opts.nocapture, + |harness| (benchfn.clone())(harness)); } DynTestFn(f) => { let cb = move || { @@ -1436,9 +1448,10 @@ pub fn run_test(opts: &TestOpts, }; run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb)) } - StaticTestFn(f) => + StaticTestFn(f) => { run_test_inner(desc, monitor_ch, opts.nocapture, - Box::new(move || __rust_begin_short_backtrace(f))), + Box::new(move || __rust_begin_short_backtrace(f))) + } } } @@ -1452,12 +1465,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box>) -> Tes match (&desc.should_panic, task_result) { (&ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TrOk, - (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => + (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => { if err.downcast_ref::() - .map(|e| &**e) - .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) - .map(|e| e.contains(msg)) - .unwrap_or(false) { + .map(|e| &**e) + .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) + .map(|e| e.contains(msg)) + .unwrap_or(false) + { TrOk } else { if desc.allow_fail { @@ -1465,7 +1479,8 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box>) -> Tes } else { TrFailedMsg(format!("Panic did not include expected string '{}'", msg)) } - }, + } + } _ if desc.allow_fail => TrAllowedFail, _ => TrFailed, } @@ -1493,18 +1508,15 @@ impl MetricMap { /// you want to see grow larger, so a change larger than `noise` in the /// negative direction represents a regression. pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) { - let m = Metric { - value, - noise, - }; + let m = Metric { value, noise }; self.0.insert(name.to_owned(), m); } pub fn fmt_metrics(&self) -> String { let v = self.0 - .iter() - .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) - .collect::>(); + .iter() + .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) + .collect::>(); v.join(", ") } } @@ -1534,7 +1546,8 @@ pub fn black_box(dummy: T) -> T { impl Bencher { /// Callback for benchmark functions to run in their body. pub fn iter(&mut self, mut inner: F) - where F: FnMut() -> T + where + F: FnMut() -> T, { if self.mode == BenchMode::Single { ns_iter_inner(&mut inner, 1); @@ -1545,7 +1558,8 @@ impl Bencher { } pub fn bench(&mut self, mut f: F) -> Option - where F: FnMut(&mut Bencher) + where + F: FnMut(&mut Bencher), { f(self); return self.summary; @@ -1557,7 +1571,8 @@ fn ns_from_dur(dur: Duration) -> u64 { } fn ns_iter_inner(inner: &mut F, k: u64) -> u64 - where F: FnMut() -> T +where + F: FnMut() -> T, { let start = Instant::now(); for _ in 0..k { @@ -1568,7 +1583,8 @@ fn ns_iter_inner(inner: &mut F, k: u64) -> u64 pub fn iter(inner: &mut F) -> stats::Summary - where F: FnMut() -> T +where + F: FnMut() -> T, { // Initial bench run to get ballpark figure. let ns_single = ns_iter_inner(inner, 1); @@ -1610,7 +1626,8 @@ pub fn iter(inner: &mut F) -> stats::Summary // If we've run for 100ms and seem to have converged to a // stable median. if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 && - summ.median - summ5.median < summ5.median_abs_dev { + summ.median - summ5.median < summ5.median_abs_dev + { return summ5; } @@ -1634,12 +1651,16 @@ pub fn iter(inner: &mut F) -> stats::Summary } pub mod bench { + use std::panic::{catch_unwind, AssertUnwindSafe}; use std::cmp; + use std::io; + use std::sync::{Arc, Mutex}; use stats; - use super::{Bencher, BenchSamples, BenchMode}; + use super::{Bencher, BenchSamples, BenchMode, Sink, MonitorMsg, TestDesc, Sender, TestResult}; - pub fn benchmark(f: F) -> BenchSamples - where F: FnMut(&mut Bencher) + pub fn benchmark(desc: TestDesc, monitor_ch: Sender, nocapture: bool, f: F) + where + F: FnMut(&mut Bencher), { let mut bs = Bencher { mode: BenchMode::Auto, @@ -1647,30 +1668,58 @@ pub mod bench { bytes: 0, }; - return match bs.bench(f) { - Some(ns_iter_summ) => { + let data = Arc::new(Mutex::new(Vec::new())); + let data2 = data.clone(); + + let oldio = if !nocapture { + Some(( + io::set_print(Some(Box::new(Sink(data2.clone())))), + io::set_panic(Some(Box::new(Sink(data2)))), + )) + } else { + None + }; + + let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f))); + + if let Some((printio, panicio)) = oldio { + io::set_print(printio); + io::set_panic(panicio); + }; + + let test_result = match result { //bs.bench(f) { + Ok(Some(ns_iter_summ)) => { let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); let mb_s = bs.bytes * 1000 / ns_iter; - BenchSamples { + let bs = BenchSamples { ns_iter_summ, mb_s: mb_s as usize, - } + }; + TestResult::TrBench(bs) } - None => { + Ok(None) => { // iter not called, so no data. // FIXME: error in this case? let samples: &mut [f64] = &mut [0.0_f64; 1]; - BenchSamples { + let bs = BenchSamples { ns_iter_summ: stats::Summary::new(samples), mb_s: 0, - } + }; + TestResult::TrBench(bs) + } + Err(_) => { + TestResult::TrFailed } }; + + let stdout = data.lock().unwrap().to_vec(); + monitor_ch.send((desc, test_result, stdout)).unwrap(); } pub fn run_once(f: F) - where F: FnMut(&mut Bencher) + where + F: FnMut(&mut Bencher), { let mut bs = Bencher { mode: BenchMode::Single, @@ -1810,7 +1859,11 @@ mod tests { #[test] fn parse_ignored_flag() { - let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()]; + let args = vec![ + "progname".to_string(), + "filter".to_string(), + "--ignored".to_string(), + ]; let opts = match parse_opts(&args) { Some(Ok(o)) => o, _ => panic!("Malformed arg in parse_ignored_flag"), @@ -1827,7 +1880,8 @@ mod tests { opts.run_tests = true; opts.run_ignored = true; - let tests = vec![TestDescAndFn { + let tests = + vec![TestDescAndFn { desc: TestDesc { name: StaticTestName("1"), ignore: true, @@ -1855,72 +1909,95 @@ mod tests { #[test] pub fn exact_filter_match() { fn tests() -> Vec { - vec!["base", - "base::test", - "base::test1", - "base::test2", - ].into_iter() - .map(|name| TestDescAndFn { - desc: TestDesc { - name: StaticTestName(name), - ignore: false, - should_panic: ShouldPanic::No, - allow_fail: false, - }, - testfn: DynTestFn(Box::new(move || {})) - }) - .collect() + vec!["base", "base::test", "base::test1", "base::test2"] + .into_iter() + .map(|name| { + TestDescAndFn { + desc: TestDesc { + name: StaticTestName(name), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(move || {})) + } + }).collect() } - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("base".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 4); - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("bas".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 4); - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("::test".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 3); - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("base::test".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 3); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("base".into()), - filter_exact: true, ..TestOpts::new() - }, tests()); + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); assert_eq!(exact.len(), 1); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("bas".into()), filter_exact: true, ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(exact.len(), 0); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("::test".into()), filter_exact: true, ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(exact.len(), 0); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("base::test".into()), filter_exact: true, ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(exact.len(), 1); } @@ -1929,15 +2006,17 @@ mod tests { let mut opts = TestOpts::new(); opts.run_tests = true; - let names = vec!["sha1::test".to_string(), - "isize::test_to_str".to_string(), - "isize::test_pow".to_string(), - "test::do_not_run_ignored_tests".to_string(), - "test::ignored_tests_result_in_ignored".to_string(), - "test::first_free_arg_should_be_a_filter".to_string(), - "test::parse_ignored_flag".to_string(), - "test::filter_for_ignored_option".to_string(), - "test::sort_tests".to_string()]; + let names = vec![ + "sha1::test".to_string(), + "isize::test_to_str".to_string(), + "isize::test_pow".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::parse_ignored_flag".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::sort_tests".to_string(), + ]; let tests = { fn testfn() {} let mut tests = Vec::new(); @@ -1957,15 +2036,17 @@ mod tests { }; let filtered = filter_tests(&opts, tests); - let expected = vec!["isize::test_pow".to_string(), - "isize::test_to_str".to_string(), - "sha1::test".to_string(), - "test::do_not_run_ignored_tests".to_string(), - "test::filter_for_ignored_option".to_string(), - "test::first_free_arg_should_be_a_filter".to_string(), - "test::ignored_tests_result_in_ignored".to_string(), - "test::parse_ignored_flag".to_string(), - "test::sort_tests".to_string()]; + let expected = vec![ + "isize::test_pow".to_string(), + "isize::test_to_str".to_string(), + "sha1::test".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::parse_ignored_flag".to_string(), + "test::sort_tests".to_string(), + ]; for (a, b) in expected.iter().zip(filtered) { assert!(*a == b.desc.name.to_string()); @@ -2004,8 +2085,7 @@ mod tests { #[test] pub fn test_bench_once_iter() { fn f(b: &mut Bencher) { - b.iter(|| { - }) + b.iter(|| {}) } bench::run_once(f); } @@ -2013,15 +2093,42 @@ mod tests { #[test] pub fn test_bench_no_iter() { fn f(_: &mut Bencher) {} - bench::benchmark(f); + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + ::bench::benchmark(desc, + tx, + true, + f); + rx.recv().unwrap(); } #[test] pub fn test_bench_iter() { fn f(b: &mut Bencher) { - b.iter(|| { - }) + b.iter(|| {}) } - bench::benchmark(f); + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + ::bench::benchmark(desc, + tx, + true, + f); + rx.recv().unwrap(); } } diff --git a/src/libtest/stats.rs b/src/libtest/stats.rs index 9f8b4a73d0cc0..e22fdf77fc171 100644 --- a/src/libtest/stats.rs +++ b/src/libtest/stats.rs @@ -400,16 +400,18 @@ mod tests { } #[test] fn test_norm10narrow() { - let val = &[966.0000000000, - 985.0000000000, - 1110.0000000000, - 848.0000000000, - 821.0000000000, - 975.0000000000, - 962.0000000000, - 1157.0000000000, - 1217.0000000000, - 955.0000000000]; + let val = &[ + 966.0000000000, + 985.0000000000, + 1110.0000000000, + 848.0000000000, + 821.0000000000, + 975.0000000000, + 962.0000000000, + 1157.0000000000, + 1217.0000000000, + 955.0000000000, + ]; let summ = &Summary { sum: 9996.0000000000, min: 821.0000000000, @@ -428,16 +430,18 @@ mod tests { } #[test] fn test_norm10medium() { - let val = &[954.0000000000, - 1064.0000000000, - 855.0000000000, - 1000.0000000000, - 743.0000000000, - 1084.0000000000, - 704.0000000000, - 1023.0000000000, - 357.0000000000, - 869.0000000000]; + let val = &[ + 954.0000000000, + 1064.0000000000, + 855.0000000000, + 1000.0000000000, + 743.0000000000, + 1084.0000000000, + 704.0000000000, + 1023.0000000000, + 357.0000000000, + 869.0000000000, + ]; let summ = &Summary { sum: 8653.0000000000, min: 357.0000000000, @@ -456,16 +460,18 @@ mod tests { } #[test] fn test_norm10wide() { - let val = &[505.0000000000, - 497.0000000000, - 1591.0000000000, - 887.0000000000, - 1026.0000000000, - 136.0000000000, - 1580.0000000000, - 940.0000000000, - 754.0000000000, - 1433.0000000000]; + let val = &[ + 505.0000000000, + 497.0000000000, + 1591.0000000000, + 887.0000000000, + 1026.0000000000, + 136.0000000000, + 1580.0000000000, + 940.0000000000, + 754.0000000000, + 1433.0000000000, + ]; let summ = &Summary { sum: 9349.0000000000, min: 136.0000000000, @@ -484,31 +490,33 @@ mod tests { } #[test] fn test_norm25verynarrow() { - let val = &[991.0000000000, - 1018.0000000000, - 998.0000000000, - 1013.0000000000, - 974.0000000000, - 1007.0000000000, - 1014.0000000000, - 999.0000000000, - 1011.0000000000, - 978.0000000000, - 985.0000000000, - 999.0000000000, - 983.0000000000, - 982.0000000000, - 1015.0000000000, - 1002.0000000000, - 977.0000000000, - 948.0000000000, - 1040.0000000000, - 974.0000000000, - 996.0000000000, - 989.0000000000, - 1015.0000000000, - 994.0000000000, - 1024.0000000000]; + let val = &[ + 991.0000000000, + 1018.0000000000, + 998.0000000000, + 1013.0000000000, + 974.0000000000, + 1007.0000000000, + 1014.0000000000, + 999.0000000000, + 1011.0000000000, + 978.0000000000, + 985.0000000000, + 999.0000000000, + 983.0000000000, + 982.0000000000, + 1015.0000000000, + 1002.0000000000, + 977.0000000000, + 948.0000000000, + 1040.0000000000, + 974.0000000000, + 996.0000000000, + 989.0000000000, + 1015.0000000000, + 994.0000000000, + 1024.0000000000, + ]; let summ = &Summary { sum: 24926.0000000000, min: 948.0000000000, @@ -527,16 +535,18 @@ mod tests { } #[test] fn test_exp10a() { - let val = &[23.0000000000, - 11.0000000000, - 2.0000000000, - 57.0000000000, - 4.0000000000, - 12.0000000000, - 5.0000000000, - 29.0000000000, - 3.0000000000, - 21.0000000000]; + let val = &[ + 23.0000000000, + 11.0000000000, + 2.0000000000, + 57.0000000000, + 4.0000000000, + 12.0000000000, + 5.0000000000, + 29.0000000000, + 3.0000000000, + 21.0000000000, + ]; let summ = &Summary { sum: 167.0000000000, min: 2.0000000000, @@ -555,16 +565,18 @@ mod tests { } #[test] fn test_exp10b() { - let val = &[24.0000000000, - 17.0000000000, - 6.0000000000, - 38.0000000000, - 25.0000000000, - 7.0000000000, - 51.0000000000, - 2.0000000000, - 61.0000000000, - 32.0000000000]; + let val = &[ + 24.0000000000, + 17.0000000000, + 6.0000000000, + 38.0000000000, + 25.0000000000, + 7.0000000000, + 51.0000000000, + 2.0000000000, + 61.0000000000, + 32.0000000000, + ]; let summ = &Summary { sum: 263.0000000000, min: 2.0000000000, @@ -583,16 +595,18 @@ mod tests { } #[test] fn test_exp10c() { - let val = &[71.0000000000, - 2.0000000000, - 32.0000000000, - 1.0000000000, - 6.0000000000, - 28.0000000000, - 13.0000000000, - 37.0000000000, - 16.0000000000, - 36.0000000000]; + let val = &[ + 71.0000000000, + 2.0000000000, + 32.0000000000, + 1.0000000000, + 6.0000000000, + 28.0000000000, + 13.0000000000, + 37.0000000000, + 16.0000000000, + 36.0000000000, + ]; let summ = &Summary { sum: 242.0000000000, min: 1.0000000000, @@ -611,31 +625,33 @@ mod tests { } #[test] fn test_exp25() { - let val = &[3.0000000000, - 24.0000000000, - 1.0000000000, - 19.0000000000, - 7.0000000000, - 5.0000000000, - 30.0000000000, - 39.0000000000, - 31.0000000000, - 13.0000000000, - 25.0000000000, - 48.0000000000, - 1.0000000000, - 6.0000000000, - 42.0000000000, - 63.0000000000, - 2.0000000000, - 12.0000000000, - 108.0000000000, - 26.0000000000, - 1.0000000000, - 7.0000000000, - 44.0000000000, - 25.0000000000, - 11.0000000000]; + let val = &[ + 3.0000000000, + 24.0000000000, + 1.0000000000, + 19.0000000000, + 7.0000000000, + 5.0000000000, + 30.0000000000, + 39.0000000000, + 31.0000000000, + 13.0000000000, + 25.0000000000, + 48.0000000000, + 1.0000000000, + 6.0000000000, + 42.0000000000, + 63.0000000000, + 2.0000000000, + 12.0000000000, + 108.0000000000, + 26.0000000000, + 1.0000000000, + 7.0000000000, + 44.0000000000, + 25.0000000000, + 11.0000000000, + ]; let summ = &Summary { sum: 593.0000000000, min: 1.0000000000, @@ -654,31 +670,33 @@ mod tests { } #[test] fn test_binom25() { - let val = &[18.0000000000, - 17.0000000000, - 27.0000000000, - 15.0000000000, - 21.0000000000, - 25.0000000000, - 17.0000000000, - 24.0000000000, - 25.0000000000, - 24.0000000000, - 26.0000000000, - 26.0000000000, - 23.0000000000, - 15.0000000000, - 23.0000000000, - 17.0000000000, - 18.0000000000, - 18.0000000000, - 21.0000000000, - 16.0000000000, - 15.0000000000, - 31.0000000000, - 20.0000000000, - 17.0000000000, - 15.0000000000]; + let val = &[ + 18.0000000000, + 17.0000000000, + 27.0000000000, + 15.0000000000, + 21.0000000000, + 25.0000000000, + 17.0000000000, + 24.0000000000, + 25.0000000000, + 24.0000000000, + 26.0000000000, + 26.0000000000, + 23.0000000000, + 15.0000000000, + 23.0000000000, + 17.0000000000, + 18.0000000000, + 18.0000000000, + 21.0000000000, + 16.0000000000, + 15.0000000000, + 31.0000000000, + 20.0000000000, + 17.0000000000, + 15.0000000000, + ]; let summ = &Summary { sum: 514.0000000000, min: 15.0000000000, @@ -697,31 +715,33 @@ mod tests { } #[test] fn test_pois25lambda30() { - let val = &[27.0000000000, - 33.0000000000, - 34.0000000000, - 34.0000000000, - 24.0000000000, - 39.0000000000, - 28.0000000000, - 27.0000000000, - 31.0000000000, - 28.0000000000, - 38.0000000000, - 21.0000000000, - 33.0000000000, - 36.0000000000, - 29.0000000000, - 37.0000000000, - 32.0000000000, - 34.0000000000, - 31.0000000000, - 39.0000000000, - 25.0000000000, - 31.0000000000, - 32.0000000000, - 40.0000000000, - 24.0000000000]; + let val = &[ + 27.0000000000, + 33.0000000000, + 34.0000000000, + 34.0000000000, + 24.0000000000, + 39.0000000000, + 28.0000000000, + 27.0000000000, + 31.0000000000, + 28.0000000000, + 38.0000000000, + 21.0000000000, + 33.0000000000, + 36.0000000000, + 29.0000000000, + 37.0000000000, + 32.0000000000, + 34.0000000000, + 31.0000000000, + 39.0000000000, + 25.0000000000, + 31.0000000000, + 32.0000000000, + 40.0000000000, + 24.0000000000, + ]; let summ = &Summary { sum: 787.0000000000, min: 21.0000000000, @@ -740,31 +760,33 @@ mod tests { } #[test] fn test_pois25lambda40() { - let val = &[42.0000000000, - 50.0000000000, - 42.0000000000, - 46.0000000000, - 34.0000000000, - 45.0000000000, - 34.0000000000, - 49.0000000000, - 39.0000000000, - 28.0000000000, - 40.0000000000, - 35.0000000000, - 37.0000000000, - 39.0000000000, - 46.0000000000, - 44.0000000000, - 32.0000000000, - 45.0000000000, - 42.0000000000, - 37.0000000000, - 48.0000000000, - 42.0000000000, - 33.0000000000, - 42.0000000000, - 48.0000000000]; + let val = &[ + 42.0000000000, + 50.0000000000, + 42.0000000000, + 46.0000000000, + 34.0000000000, + 45.0000000000, + 34.0000000000, + 49.0000000000, + 39.0000000000, + 28.0000000000, + 40.0000000000, + 35.0000000000, + 37.0000000000, + 39.0000000000, + 46.0000000000, + 44.0000000000, + 32.0000000000, + 45.0000000000, + 42.0000000000, + 37.0000000000, + 48.0000000000, + 42.0000000000, + 33.0000000000, + 42.0000000000, + 48.0000000000, + ]; let summ = &Summary { sum: 1019.0000000000, min: 28.0000000000, @@ -783,31 +805,33 @@ mod tests { } #[test] fn test_pois25lambda50() { - let val = &[45.0000000000, - 43.0000000000, - 44.0000000000, - 61.0000000000, - 51.0000000000, - 53.0000000000, - 59.0000000000, - 52.0000000000, - 49.0000000000, - 51.0000000000, - 51.0000000000, - 50.0000000000, - 49.0000000000, - 56.0000000000, - 42.0000000000, - 52.0000000000, - 51.0000000000, - 43.0000000000, - 48.0000000000, - 48.0000000000, - 50.0000000000, - 42.0000000000, - 43.0000000000, - 42.0000000000, - 60.0000000000]; + let val = &[ + 45.0000000000, + 43.0000000000, + 44.0000000000, + 61.0000000000, + 51.0000000000, + 53.0000000000, + 59.0000000000, + 52.0000000000, + 49.0000000000, + 51.0000000000, + 51.0000000000, + 50.0000000000, + 49.0000000000, + 56.0000000000, + 42.0000000000, + 52.0000000000, + 51.0000000000, + 43.0000000000, + 48.0000000000, + 48.0000000000, + 50.0000000000, + 42.0000000000, + 43.0000000000, + 42.0000000000, + 60.0000000000, + ]; let summ = &Summary { sum: 1235.0000000000, min: 42.0000000000, @@ -826,31 +850,33 @@ mod tests { } #[test] fn test_unif25() { - let val = &[99.0000000000, - 55.0000000000, - 92.0000000000, - 79.0000000000, - 14.0000000000, - 2.0000000000, - 33.0000000000, - 49.0000000000, - 3.0000000000, - 32.0000000000, - 84.0000000000, - 59.0000000000, - 22.0000000000, - 86.0000000000, - 76.0000000000, - 31.0000000000, - 29.0000000000, - 11.0000000000, - 41.0000000000, - 53.0000000000, - 45.0000000000, - 44.0000000000, - 98.0000000000, - 98.0000000000, - 7.0000000000]; + let val = &[ + 99.0000000000, + 55.0000000000, + 92.0000000000, + 79.0000000000, + 14.0000000000, + 2.0000000000, + 33.0000000000, + 49.0000000000, + 3.0000000000, + 32.0000000000, + 84.0000000000, + 59.0000000000, + 22.0000000000, + 86.0000000000, + 76.0000000000, + 31.0000000000, + 29.0000000000, + 11.0000000000, + 41.0000000000, + 53.0000000000, + 45.0000000000, + 44.0000000000, + 98.0000000000, + 98.0000000000, + 7.0000000000, + ]; let summ = &Summary { sum: 1242.0000000000, min: 2.0000000000, @@ -885,18 +911,14 @@ mod bench { #[bench] pub fn sum_three_items(b: &mut Bencher) { - b.iter(|| { - [1e20f64, 1.5f64, -1e20f64].sum(); - }) + b.iter(|| { [1e20f64, 1.5f64, -1e20f64].sum(); }) } #[bench] pub fn sum_many_f64(b: &mut Bencher) { let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60]; let v = (0..500).map(|i| nums[i % 5]).collect::>(); - b.iter(|| { - v.sum(); - }) + b.iter(|| { v.sum(); }) } #[bench] diff --git a/src/test/run-make/libtest-json/Makefile b/src/test/run-make/libtest-json/Makefile new file mode 100644 index 0000000000000..ec91ddfb9f917 --- /dev/null +++ b/src/test/run-make/libtest-json/Makefile @@ -0,0 +1,14 @@ +-include ../tools.mk + +# Test expected libtest's JSON output + +OUTPUT_FILE := $(TMPDIR)/libtest-json-output.json + +all: + $(RUSTC) --test f.rs + $(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE) || true + + cat $(OUTPUT_FILE) | "$(PYTHON)" validate_json.py + + # Compare to output file + diff output.json $(OUTPUT_FILE) diff --git a/src/test/run-make/libtest-json/f.rs b/src/test/run-make/libtest-json/f.rs new file mode 100644 index 0000000000000..5cff1f1a5b1af --- /dev/null +++ b/src/test/run-make/libtest-json/f.rs @@ -0,0 +1,32 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[test] +fn a() { + // Should pass +} + +#[test] +fn b() { + assert!(false) +} + +#[test] +#[should_panic] +fn c() { + assert!(false); +} + +#[test] +#[ignore] +fn d() { + assert!(false); +} + diff --git a/src/test/run-make/libtest-json/output.json b/src/test/run-make/libtest-json/output.json new file mode 100644 index 0000000000000..235f8cd7c7257 --- /dev/null +++ b/src/test/run-make/libtest-json/output.json @@ -0,0 +1,10 @@ +{ "type": "suite", "event": "started", "test_count": "4" } +{ "type": "test", "event": "started", "name": "a" } +{ "type": "test", "name": "a", "event": "ok" } +{ "type": "test", "event": "started", "name": "b" } +{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:5\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" } +{ "type": "test", "event": "started", "name": "c" } +{ "type": "test", "name": "c", "event": "ok" } +{ "type": "test", "event": "started", "name": "d" } +{ "type": "test", "name": "d", "event": "ignored" } +{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": "0" } diff --git a/src/test/run-make/libtest-json/validate_json.py b/src/test/run-make/libtest-json/validate_json.py new file mode 100755 index 0000000000000..1e97639b524e8 --- /dev/null +++ b/src/test/run-make/libtest-json/validate_json.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import sys +import json + +# Try to decode line in order to ensure it is a valid JSON document +for line in sys.stdin: + json.loads(line) diff --git a/src/tools/cargo b/src/tools/cargo index 91e36aa86c703..1d6dfea44f971 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 91e36aa86c7037de50642f2fec1cf47c3d18af02 +Subproject commit 1d6dfea44f97199d5d5c177c7dadcde393eaff9a diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 533aaf9cd2735..1c52ebd7dc54e 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -485,7 +485,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts { filter: config.filter.clone(), filter_exact: config.filter_exact, run_ignored: config.run_ignored, - quiet: config.quiet, + format: if config.quiet { test::OutputFormat::Terse } else { test::OutputFormat::Pretty }, logfile: config.logfile.clone(), run_tests: true, bench_benchmarks: true,