From c4aa7a71a9cf16a242530fc1c3aedffb8567cba3 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Tue, 13 Aug 2024 12:55:58 +1000 Subject: [PATCH 1/2] Port `run-make/libtest-json` to rmake --- .../tidy/src/allowed_run_make_makefiles.txt | 1 - tests/run-make/libtest-json/Makefile | 20 ------------ .../run-make/libtest-json/output-default.json | 2 +- .../libtest-json/output-stdout-success.json | 2 +- tests/run-make/libtest-json/rmake.rs | 31 +++++++++++++++++++ 5 files changed, 33 insertions(+), 23 deletions(-) delete mode 100644 tests/run-make/libtest-json/Makefile create mode 100644 tests/run-make/libtest-json/rmake.rs diff --git a/src/tools/tidy/src/allowed_run_make_makefiles.txt b/src/tools/tidy/src/allowed_run_make_makefiles.txt index 14f0a9cd23d21..96ab097b3885e 100644 --- a/src/tools/tidy/src/allowed_run_make_makefiles.txt +++ b/src/tools/tidy/src/allowed_run_make_makefiles.txt @@ -9,7 +9,6 @@ run-make/incr-add-rust-src-component/Makefile run-make/issue-84395-lto-embed-bitcode/Makefile run-make/jobserver-error/Makefile run-make/libs-through-symlinks/Makefile -run-make/libtest-json/Makefile run-make/libtest-junit/Makefile run-make/libtest-thread-limit/Makefile run-make/macos-deployment-target/Makefile diff --git a/tests/run-make/libtest-json/Makefile b/tests/run-make/libtest-json/Makefile deleted file mode 100644 index c8bc7b5dd4a4c..0000000000000 --- a/tests/run-make/libtest-json/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# ignore-cross-compile -# needs-unwind -include ../tools.mk - -# Test expected libtest's JSON output - -OUTPUT_FILE_DEFAULT := $(TMPDIR)/libtest-json-output-default.json -OUTPUT_FILE_STDOUT_SUCCESS := $(TMPDIR)/libtest-json-output-stdout-success.json - -all: f.rs validate_json.py output-default.json output-stdout-success.json - $(RUSTC) --test f.rs - RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE_DEFAULT) || true - RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=json --show-output > $(OUTPUT_FILE_STDOUT_SUCCESS) || true - - cat $(OUTPUT_FILE_DEFAULT) | "$(PYTHON)" validate_json.py - cat $(OUTPUT_FILE_STDOUT_SUCCESS) | "$(PYTHON)" validate_json.py - - # Normalize the actual output and compare to expected output file - cat $(OUTPUT_FILE_DEFAULT) | sed 's/"exec_time": [0-9.]*/"exec_time": $$TIME/' | diff output-default.json - - cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed 's/"exec_time": [0-9.]*/"exec_time": $$TIME/' | diff output-stdout-success.json - diff --git a/tests/run-make/libtest-json/output-default.json b/tests/run-make/libtest-json/output-default.json index 01710f59e5d74..a2293a032d012 100644 --- a/tests/run-make/libtest-json/output-default.json +++ b/tests/run-make/libtest-json/output-default.json @@ -7,4 +7,4 @@ { "type": "test", "name": "c", "event": "ok" } { "type": "test", "event": "started", "name": "d" } { "type": "test", "name": "d", "event": "ignored", "message": "msg" } -{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": $TIME } +{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": "$EXEC_TIME" } diff --git a/tests/run-make/libtest-json/output-stdout-success.json b/tests/run-make/libtest-json/output-stdout-success.json index 878eb6c7c260d..cf92f01f63aca 100644 --- a/tests/run-make/libtest-json/output-stdout-success.json +++ b/tests/run-make/libtest-json/output-stdout-success.json @@ -7,4 +7,4 @@ { "type": "test", "name": "c", "event": "ok", "stdout": "thread 'c' panicked at f.rs:15:5:\nassertion failed: false\n" } { "type": "test", "event": "started", "name": "d" } { "type": "test", "name": "d", "event": "ignored", "message": "msg" } -{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": $TIME } +{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": "$EXEC_TIME" } diff --git a/tests/run-make/libtest-json/rmake.rs b/tests/run-make/libtest-json/rmake.rs new file mode 100644 index 0000000000000..acbd88dc46cb9 --- /dev/null +++ b/tests/run-make/libtest-json/rmake.rs @@ -0,0 +1,31 @@ +// Check libtest's JSON output against snapshots. + +//@ ignore-cross-compile +//@ needs-unwind (test file contains #[should_panic] test) + +use run_make_support::{cmd, diff, python_command, rustc}; + +fn main() { + rustc().arg("--test").input("f.rs").run(); + + run_tests(&[], "output-default.json"); + run_tests(&["--show-output"], "output-stdout-success.json"); +} + +#[track_caller] +fn run_tests(extra_args: &[&str], expected_file: &str) { + let cmd_out = cmd("./f") + .env("RUST_BACKTRACE", "0") + .args(&["-Zunstable-options", "--test-threads=1", "--format=json"]) + .args(extra_args) + .run_fail(); + let test_stdout = &cmd_out.stdout_utf8(); + + python_command().arg("validate_json.py").stdin(test_stdout).run(); + + diff() + .expected_file(expected_file) + .actual_text("stdout", test_stdout) + .normalize(r#"(?"exec_time": )[0-9.]+"#, r#"${prefix}"$$EXEC_TIME""#) + .run(); +} From fc733a668afe9a16a23a6d249c6d253a08f36ebf Mon Sep 17 00:00:00 2001 From: Zalathar Date: Tue, 13 Aug 2024 13:02:26 +1000 Subject: [PATCH 2/2] Port `run-make/libtest-junit` to rmake --- .../tidy/src/allowed_run_make_makefiles.txt | 1 - tests/run-make/libtest-junit/Makefile | 20 ------------ tests/run-make/libtest-junit/rmake.rs | 31 +++++++++++++++++++ 3 files changed, 31 insertions(+), 21 deletions(-) delete mode 100644 tests/run-make/libtest-junit/Makefile create mode 100644 tests/run-make/libtest-junit/rmake.rs diff --git a/src/tools/tidy/src/allowed_run_make_makefiles.txt b/src/tools/tidy/src/allowed_run_make_makefiles.txt index 96ab097b3885e..e9dcc70dff538 100644 --- a/src/tools/tidy/src/allowed_run_make_makefiles.txt +++ b/src/tools/tidy/src/allowed_run_make_makefiles.txt @@ -9,7 +9,6 @@ run-make/incr-add-rust-src-component/Makefile run-make/issue-84395-lto-embed-bitcode/Makefile run-make/jobserver-error/Makefile run-make/libs-through-symlinks/Makefile -run-make/libtest-junit/Makefile run-make/libtest-thread-limit/Makefile run-make/macos-deployment-target/Makefile run-make/min-global-align/Makefile diff --git a/tests/run-make/libtest-junit/Makefile b/tests/run-make/libtest-junit/Makefile deleted file mode 100644 index 26e56242dd239..0000000000000 --- a/tests/run-make/libtest-junit/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# ignore-cross-compile -# needs-unwind contains should_panic test -include ../tools.mk - -# Test expected libtest's junit output - -OUTPUT_FILE_DEFAULT := $(TMPDIR)/libtest-junit-output-default.xml -OUTPUT_FILE_STDOUT_SUCCESS := $(TMPDIR)/libtest-junit-output-stdout-success.xml - -all: f.rs validate_junit.py output-default.xml output-stdout-success.xml - $(RUSTC) --test f.rs - RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=junit > $(OUTPUT_FILE_DEFAULT) || true - RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=junit --show-output > $(OUTPUT_FILE_STDOUT_SUCCESS) || true - - cat $(OUTPUT_FILE_DEFAULT) | "$(PYTHON)" validate_junit.py - cat $(OUTPUT_FILE_STDOUT_SUCCESS) | "$(PYTHON)" validate_junit.py - - # Normalize the actual output and compare to expected output file - cat $(OUTPUT_FILE_DEFAULT) | sed 's/time="[0-9.]*"/time="$$TIME"/g' | diff output-default.xml - - cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed 's/time="[0-9.]*"/time="$$TIME"/g' | diff output-stdout-success.xml - diff --git a/tests/run-make/libtest-junit/rmake.rs b/tests/run-make/libtest-junit/rmake.rs new file mode 100644 index 0000000000000..d631313ed9210 --- /dev/null +++ b/tests/run-make/libtest-junit/rmake.rs @@ -0,0 +1,31 @@ +// Check libtest's JUnit (XML) output against snapshots. + +//@ ignore-cross-compile +//@ needs-unwind (test file contains #[should_panic] test) + +use run_make_support::{cmd, diff, python_command, rustc}; + +fn main() { + rustc().arg("--test").input("f.rs").run(); + + run_tests(&[], "output-default.xml"); + run_tests(&["--show-output"], "output-stdout-success.xml"); +} + +#[track_caller] +fn run_tests(extra_args: &[&str], expected_file: &str) { + let cmd_out = cmd("./f") + .env("RUST_BACKTRACE", "0") + .args(&["-Zunstable-options", "--test-threads=1", "--format=junit"]) + .args(extra_args) + .run_fail(); + let test_stdout = &cmd_out.stdout_utf8(); + + python_command().arg("validate_junit.py").stdin(test_stdout).run(); + + diff() + .expected_file(expected_file) + .actual_text("stdout", test_stdout) + .normalize(r#"\btime="[0-9.]+""#, r#"time="$$TIME""#) + .run(); +}