diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index 6c2c05ac7197e..84d69e52de635 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -1039,6 +1039,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the
         cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite));
         cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite));
         cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target));
+        cmd.arg("--suite").arg(suite);
         cmd.arg("--mode").arg(mode);
         cmd.arg("--target").arg(target.rustc_target_arg());
         cmd.arg("--host").arg(&*compiler.host.triple);
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index 2f832b53a9039..24ef98cd784d8 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -224,6 +224,10 @@ pub struct Config {
     /// The test mode, compile-fail, run-fail, ui
     pub mode: Mode,
 
+    /// The test suite (essentially which directory is running, but without the
+    /// directory prefix such as src/test)
+    pub suite: String,
+
     /// The debugger to use in debuginfo mode. Unset otherwise.
     pub debugger: Option<Debugger>,
 
diff --git a/src/tools/compiletest/src/header/tests.rs b/src/tools/compiletest/src/header/tests.rs
index 1f82b137ee6cf..4bcbd89f09510 100644
--- a/src/tools/compiletest/src/header/tests.rs
+++ b/src/tools/compiletest/src/header/tests.rs
@@ -39,6 +39,7 @@ fn config() -> Config {
     let args = &[
         "compiletest",
         "--mode=ui",
+        "--suite=ui",
         "--compile-lib-path=",
         "--run-lib-path=",
         "--rustc-path=",
diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs
index 2b2a6cfa8bd6f..2b167ac8e9fb2 100644
--- a/src/tools/compiletest/src/main.rs
+++ b/src/tools/compiletest/src/main.rs
@@ -70,6 +70,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "compile-fail | run-fail | run-pass-valgrind | pretty | debug-info | codegen | rustdoc \
              codegen-units | incremental | run-make | ui | js-doc-test | mir-opt | assembly",
         )
+        .reqopt(
+            "",
+            "suite",
+            "which suite of compile tests to run. used for nicer error reporting.",
+            "SUITE",
+        )
         .optopt(
             "",
             "pass",
@@ -201,6 +207,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         build_base: opt_path(matches, "build-base"),
         stage_id: matches.opt_str("stage-id").unwrap(),
         mode: matches.opt_str("mode").unwrap().parse().expect("invalid mode"),
+        suite: matches.opt_str("suite").unwrap(),
         debugger: None,
         run_ignored,
         filter: matches.free.first().cloned(),
@@ -340,7 +347,7 @@ pub fn run_tests(config: Config) {
             configs.extend(configure_lldb(&config));
         }
     } else {
-        configs.push(config);
+        configs.push(config.clone());
     };
 
     let mut tests = Vec::new();
@@ -351,11 +358,32 @@ pub fn run_tests(config: Config) {
     let res = test::run_tests_console(&opts, tests);
     match res {
         Ok(true) => {}
-        Ok(false) => panic!("Some tests failed"),
+        Ok(false) => {
+            // We want to report that the tests failed, but we also want to give
+            // some indication of just what tests we were running. Especially on
+            // CI, where there can be cross-compiled tests for a lot of
+            // architectures, without this critical information it can be quite
+            // easy to miss which tests failed, and as such fail to reproduce
+            // the failure locally.
+
+            eprintln!(
+                "Some tests failed in compiletest suite={}{} mode={} host={} target={}",
+                config.suite,
+                config.compare_mode.map(|c| format!(" compare_mode={:?}", c)).unwrap_or_default(),
+                config.mode,
+                config.host,
+                config.target
+            );
+
+            std::process::exit(1);
+        }
         Err(e) => {
             // We don't know if tests passed or not, but if there was an error
             // during testing we don't want to just suceeed (we may not have
             // tested something), so fail.
+            //
+            // This should realistically "never" happen, so don't try to make
+            // this a pretty error message.
             panic!("I/O failure during tests: {:?}", e);
         }
     }