Auto merge of #49337 - kennytm:rollup, r=kennytm

Rollup of 21 pull requests

- Successful merges: #49046, #49076, #49089, #49120, #49121, #49122, #49162, #49193, #49194, #49203, #49229, #49235, #49254, #49268, #49273, #49274, #49290, #49312, #49314, #49318, #49299

config.toml.example

@@ -271,6 +271,9 @@
 # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE)
 #backtrace = true
 
+# Build rustc with experimental parallelization
+#experimental-parallel-queries = false
+
 # The default linker that will be hard-coded into the generated compiler for
 # targets that don't specify linker explicitly in their target specifications.
 # Note that this is not the linker used to link said compiler.

src/bootstrap/builder.rs

@@ -321,11 +321,13 @@ impl<'a> Builder<'a> {
                 test::RunMake),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
             Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook,
-                doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon,
-                doc::Reference, doc::Rustdoc, doc::RustByExample, doc::CargoBook),
-            Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts,
-                dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo,
-                dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign),
+                doc::Standalone, doc::Std, doc::Test, doc::WhitelistedRustc, doc::Rustc,
+                doc::ErrorIndex, doc::Nomicon, doc::Reference, doc::Rustdoc, doc::RustByExample,
+                doc::CargoBook),
+            Kind::Dist => describe!(dist::Docs, dist::RustcDocs, dist::Mingw, dist::Rustc,
+                dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src,
+                dist::PlainSourceTarball, dist::Cargo, dist::Rls, dist::Rustfmt, dist::Extended,
+                dist::HashSign),
             Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls,
                 install::Rustfmt, install::Analysis, install::Src, install::Rustc),
         }

src/bootstrap/check.rs

@@ -40,17 +40,18 @@ impl Step for Std {
         let target = self.target;
         let compiler = builder.compiler(0, build.build);
 
-        let _folder = build.fold_output(|| format!("stage{}-std", compiler.stage));
-        println!("Checking std artifacts ({} -> {})", &compiler.host, target);
-
         let out_dir = build.stage_out(compiler, Mode::Libstd);
         build.clear_if_dirty(&out_dir, &builder.rustc(compiler));
         let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "check");
         std_cargo(builder, &compiler, target, &mut cargo);
+
+        let _folder = build.fold_output(|| format!("stage{}-std", compiler.stage));
+        println!("Checking std artifacts ({} -> {})", &compiler.host, target);
         run_cargo(build,
                   &mut cargo,
                   &libstd_stamp(build, compiler, target),
                   true);
+
         let libdir = builder.sysroot_libdir(compiler, target);
         add_to_sysroot(&libdir, &libstd_stamp(build, compiler, target));
     }
@@ -86,19 +87,20 @@ impl Step for Rustc {
         let compiler = builder.compiler(0, build.build);
         let target = self.target;
 
-        let _folder = build.fold_output(|| format!("stage{}-rustc", compiler.stage));
-        println!("Checking compiler artifacts ({} -> {})", &compiler.host, target);
-
         let stage_out = builder.stage_out(compiler, Mode::Librustc);
         build.clear_if_dirty(&stage_out, &libstd_stamp(build, compiler, target));
         build.clear_if_dirty(&stage_out, &libtest_stamp(build, compiler, target));
 
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "check");
         rustc_cargo(build, &mut cargo);
+
+        let _folder = build.fold_output(|| format!("stage{}-rustc", compiler.stage));
+        println!("Checking compiler artifacts ({} -> {})", &compiler.host, target);
         run_cargo(build,
                   &mut cargo,
                   &librustc_stamp(build, compiler, target),
                   true);
+
         let libdir = builder.sysroot_libdir(compiler, target);
         add_to_sysroot(&libdir, &librustc_stamp(build, compiler, target));
     }
@@ -128,16 +130,18 @@ impl Step for Test {
         let target = self.target;
         let compiler = builder.compiler(0, build.build);
 
-        let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage));
-        println!("Checking test artifacts ({} -> {})", &compiler.host, target);
         let out_dir = build.stage_out(compiler, Mode::Libtest);
         build.clear_if_dirty(&out_dir, &libstd_stamp(build, compiler, target));
         let mut cargo = builder.cargo(compiler, Mode::Libtest, target, "check");
         test_cargo(build, &compiler, target, &mut cargo);
+
+        let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage));
+        println!("Checking test artifacts ({} -> {})", &compiler.host, target);
         run_cargo(build,
                   &mut cargo,
                   &libtest_stamp(build, compiler, target),
                   true);
+
         let libdir = builder.sysroot_libdir(compiler, target);
         add_to_sysroot(&libdir, &libtest_stamp(build, compiler, target));
     }

src/bootstrap/compile.rs

@@ -93,10 +93,6 @@ impl Step for Std {
             return;
         }
 
-        let _folder = build.fold_output(|| format!("stage{}-std", compiler.stage));
-        println!("Building stage{} std artifacts ({} -> {})", compiler.stage,
-                &compiler.host, target);
-
         if target.contains("musl") {
             let libdir = builder.sysroot_libdir(compiler, target);
             copy_musl_third_party_objects(build, target, &libdir);
@@ -106,6 +102,10 @@ impl Step for Std {
         build.clear_if_dirty(&out_dir, &builder.rustc(compiler));
         let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "build");
         std_cargo(builder, &compiler, target, &mut cargo);
+
+        let _folder = build.fold_output(|| format!("stage{}-std", compiler.stage));
+        println!("Building stage{} std artifacts ({} -> {})", compiler.stage,
+                &compiler.host, target);
         run_cargo(build,
                   &mut cargo,
                   &libstd_stamp(build, compiler, target),
@@ -360,13 +360,14 @@ impl Step for Test {
             return;
         }
 
-        let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage));
-        println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
-                &compiler.host, target);
         let out_dir = build.stage_out(compiler, Mode::Libtest);
         build.clear_if_dirty(&out_dir, &libstd_stamp(build, compiler, target));
         let mut cargo = builder.cargo(compiler, Mode::Libtest, target, "build");
         test_cargo(build, &compiler, target, &mut cargo);
+
+        let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage));
+        println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
+                &compiler.host, target);
         run_cargo(build,
                   &mut cargo,
                   &libtest_stamp(build, compiler, target),
@@ -482,16 +483,16 @@ impl Step for Rustc {
             target: build.build,
         });
 
-        let _folder = build.fold_output(|| format!("stage{}-rustc", compiler.stage));
-        println!("Building stage{} compiler artifacts ({} -> {})",
-                 compiler.stage, &compiler.host, target);
-
         let stage_out = builder.stage_out(compiler, Mode::Librustc);
         build.clear_if_dirty(&stage_out, &libstd_stamp(build, compiler, target));
         build.clear_if_dirty(&stage_out, &libtest_stamp(build, compiler, target));
 
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "build");
         rustc_cargo(build, &mut cargo);
+
+        let _folder = build.fold_output(|| format!("stage{}-rustc", compiler.stage));
+        println!("Building stage{} compiler artifacts ({} -> {})",
+                 compiler.stage, &compiler.host, target);
         run_cargo(build,
                   &mut cargo,
                   &librustc_stamp(build, compiler, target),
@@ -634,8 +635,6 @@ impl Step for CodegenBackend {
             .arg(build.src.join("src/librustc_trans/Cargo.toml"));
         rustc_cargo_env(build, &mut cargo);
 
-        let _folder = build.fold_output(|| format!("stage{}-rustc_trans", compiler.stage));
-
         match &*self.backend {
             "llvm" | "emscripten" => {
                 // Build LLVM for our target. This will implicitly build the
@@ -685,6 +684,8 @@ impl Step for CodegenBackend {
 
         let tmp_stamp = build.cargo_out(compiler, Mode::Librustc, target)
             .join(".tmp.stamp");
+
+        let _folder = build.fold_output(|| format!("stage{}-rustc_trans", compiler.stage));
         let files = run_cargo(build,
                               cargo.arg("--features").arg(features),
                               &tmp_stamp,

src/bootstrap/configure.py

@@ -44,6 +44,7 @@ def v(*args):
 o("docs", "build.docs", "build standard library documentation")
 o("compiler-docs", "build.compiler-docs", "build compiler documentation")
 o("optimize-tests", "rust.optimize-tests", "build tests with optimizations")
+o("experimental-parallel-queries", "rust.experimental-parallel-queries", "build rustc with experimental parallelization")
 o("test-miri", "rust.test-miri", "run miri's test suite")
 o("debuginfo-tests", "rust.debuginfo-tests", "build tests with debugger metadata")
 o("quiet-tests", "rust.quiet-tests", "enable quieter output when running tests")

src/bootstrap/dist.rs

@@ -102,7 +102,7 @@ impl Step for Docs {
 
         let dst = image.join("share/doc/rust/html");
         t!(fs::create_dir_all(&dst));
-        let src = build.out.join(host).join("doc");
+        let src = build.doc_out(host);
         cp_r(&src, &dst);
 
         let mut cmd = rust_installer(builder);
@@ -120,14 +120,69 @@ impl Step for Docs {
         build.run(&mut cmd);
         t!(fs::remove_dir_all(&image));
 
-        // As part of this step, *also* copy the docs directory to a directory which
-        // buildbot typically uploads.
-        if host == build.build {
-            let dst = distdir(build).join("doc").join(build.rust_package_vers());
-            t!(fs::create_dir_all(&dst));
-            cp_r(&src, &dst);
+        distdir(build).join(format!("{}-{}.tar.gz", name, host))
+    }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDocs {
+    pub stage: u32,
+    pub host: Interned<String>,
+}
+
+impl Step for RustcDocs {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/librustc")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(RustcDocs {
+            stage: run.builder.top_stage,
+            host: run.target,
+        });
+    }
+
+    /// Builds the `rustc-docs` installer component.
+    fn run(self, builder: &Builder) -> PathBuf {
+        let build = builder.build;
+        let host = self.host;
+
+        let name = pkgname(build, "rustc-docs");
+
+        println!("Dist compiler docs ({})", host);
+        if !build.config.compiler_docs {
+            println!("\tskipping - compiler docs disabled");
+            return distdir(build).join(format!("{}-{}.tar.gz", name, host));
         }
 
+        builder.default_doc(None);
+
+        let image = tmpdir(build).join(format!("{}-{}-image", name, host));
+        let _ = fs::remove_dir_all(&image);
+
+        let dst = image.join("share/doc/rust/html");
+        t!(fs::create_dir_all(&dst));
+        let src = build.compiler_doc_out(host);
+        cp_r(&src, &dst);
+
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rustc-Documentation")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=Rustc-documentation-is-installed.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(build))
+           .arg("--output-dir").arg(&distdir(build))
+           .arg(format!("--package-name={}-{}", name, host))
+           .arg("--component-name=rustc-docs")
+           .arg("--legacy-manifest-dirs=rustlib,cargo")
+           .arg("--bulk-dirs=share/doc/rust/html");
+        build.run(&mut cmd);
+        t!(fs::remove_dir_all(&image));
+
         distdir(build).join(format!("{}-{}.tar.gz", name, host))
     }
 }
@@ -1186,7 +1241,6 @@ impl Step for Rustfmt {
         let build = builder.build;
         let stage = self.stage;
         let target = self.target;
-        assert!(build.config.extended);
 
         println!("Dist Rustfmt stage{} ({})", stage, target);
         let src = build.src.join("src/tools/rustfmt");