diff --git a/Cargo.lock b/Cargo.lock index bfc2d3e066a89..83ac8396a16e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,6 +243,7 @@ dependencies = [ "anyhow", "flate2", "hex 0.4.2", + "num_cpus", "rayon", "serde", "serde_json", diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 4bc162abee6c1..707c1ff3efad9 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -193,37 +193,37 @@ impl StepDescription { ); } - if paths.is_empty() { - for (desc, should_run) in v.iter().zip(should_runs) { + if paths.is_empty() || builder.config.include_default_paths { + for (desc, should_run) in v.iter().zip(&should_runs) { if desc.default && should_run.is_really_default { for pathset in &should_run.paths { desc.maybe_run(builder, pathset); } } } - } else { - for path in paths { - // strip CurDir prefix if present - let path = match path.strip_prefix(".") { - Ok(p) => p, - Err(_) => path, - }; + } - let mut attempted_run = false; - for (desc, should_run) in v.iter().zip(&should_runs) { - if let Some(suite) = should_run.is_suite_path(path) { - attempted_run = true; - desc.maybe_run(builder, suite); - } else if let Some(pathset) = should_run.pathset_for_path(path) { - attempted_run = true; - desc.maybe_run(builder, pathset); - } - } + for path in paths { + // strip CurDir prefix if present + let path = match path.strip_prefix(".") { + Ok(p) => p, + Err(_) => path, + }; - if !attempted_run { - panic!("error: no rules matched {}", path.display()); + let mut attempted_run = false; + for (desc, should_run) in v.iter().zip(&should_runs) { + if let Some(suite) = should_run.is_suite_path(path) { + attempted_run = true; + desc.maybe_run(builder, suite); + } else if let Some(pathset) = should_run.pathset_for_path(path) { + attempted_run = true; + desc.maybe_run(builder, pathset); } } + + if !attempted_run { + panic!("error: no rules matched {}", path.display()); + } } } } @@ -462,6 +462,7 @@ impl<'a> Builder<'a> { dist::LlvmTools, dist::RustDev, dist::Extended, + dist::BuildManifest, dist::HashSign ), Kind::Install => describe!( diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6265bbaf5c22c..db82155bd6ad2 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -61,6 +61,7 @@ pub struct Config { pub profiler: bool, pub ignore_git: bool, pub exclude: Vec, + pub include_default_paths: bool, pub rustc_error_format: Option, pub json_output: bool, pub test_compare_mode: bool, @@ -532,6 +533,7 @@ impl Config { let mut config = Config::default_opts(); config.exclude = flags.exclude; + config.include_default_paths = flags.include_default_paths; config.rustc_error_format = flags.rustc_error_format; config.json_output = flags.json_output; config.on_fail = flags.on_fail; diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 3a0743da7a415..dd4cf9d595323 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2353,7 +2353,6 @@ impl Step for HashSign { cmd.arg(today.trim()); cmd.arg(addr); cmd.arg(&builder.config.channel); - cmd.arg(&builder.src); cmd.env("BUILD_MANIFEST_LEGACY", "1"); builder.create_dir(&distdir(builder)); @@ -2584,3 +2583,70 @@ impl Step for RustDev { Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) } } + +/// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the +/// release process to avoid cloning the monorepo and building stuff. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BuildManifest { + pub target: TargetSelection, +} + +impl Step for BuildManifest { + type Output = PathBuf; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let build_manifest = builder.tool_exe(Tool::BuildManifest); + + let name = pkgname(builder, "build-manifest"); + let tmp = tmpdir(builder); + + // Prepare the image. + let image = tmp.join("build-manifest-image"); + let image_bin = image.join("bin"); + let _ = fs::remove_dir_all(&image); + t!(fs::create_dir_all(&image_bin)); + builder.install(&build_manifest, &image_bin.join("build-manifest"), 0o755); + + // Prepare the overlay. + let overlay = tmp.join("build-manifest-overlay"); + let _ = fs::remove_dir_all(&overlay); + builder.create_dir(&overlay); + builder.create(&overlay.join("version"), &builder.rust_version()); + for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { + builder.install(&builder.src.join(file), &overlay, 0o644); + } + + // Create the final tarball. + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=build-manifest installed.") + .arg("--image-dir") + .arg(&image) + .arg("--work-dir") + .arg(&tmpdir(builder)) + .arg("--output-dir") + .arg(&distdir(builder)) + .arg("--non-installed-overlay") + .arg(&overlay) + .arg(format!("--package-name={}-{}", name, self.target.triple)) + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg("--component-name=build-manifest"); + + builder.run(&mut cmd); + distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple)) + } +} diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 319a0b4e611eb..c10188875fbc4 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -30,6 +30,7 @@ pub struct Flags { pub cmd: Subcommand, pub incremental: bool, pub exclude: Vec, + pub include_default_paths: bool, pub rustc_error_format: Option, pub json_output: bool, pub dry_run: bool, @@ -137,6 +138,11 @@ To learn more about a subcommand, run `./x.py -h`", opts.optmulti("", "host", "host targets to build", "HOST"); opts.optmulti("", "target", "target targets to build", "TARGET"); opts.optmulti("", "exclude", "build paths to exclude", "PATH"); + opts.optflag( + "", + "include-default-paths", + "include default paths in addition to the provided ones", + ); opts.optopt("", "on-fail", "command to run on failure", "CMD"); opts.optflag("", "dry-run", "dry run; don't build anything"); opts.optopt( @@ -618,6 +624,7 @@ Arguments: .into_iter() .map(|p| p.into()) .collect::>(), + include_default_paths: matches.opt_present("include-default-paths"), deny_warnings: parse_deny_warnings(&matches), llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map( |s| s.parse::().expect("`llvm-skip-rebuild` should be either true or false"), diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index 80c093e713eff..7c64e5a0aadc8 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -77,7 +77,6 @@ impl Step for BuildManifest { cmd.arg(today.trim()); cmd.arg(addr); cmd.arg(&builder.config.channel); - cmd.arg(&builder.src); builder.create_dir(&distdir(builder)); builder.run(&mut cmd); diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 58e2567a58f08..14700aeea05af 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -98,7 +98,9 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set rust.jemalloc -ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS +ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS \ + --include-default-paths \ + src/tools/build-manifest ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml index 4ae4dbfc06ede..4a2c710811f61 100644 --- a/src/tools/build-manifest/Cargo.toml +++ b/src/tools/build-manifest/Cargo.toml @@ -14,3 +14,4 @@ tar = "0.4.29" sha2 = "0.9.1" rayon = "1.3.1" hex = "0.4.2" +num_cpus = "1.13.0" diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index 26e96c9fd8fda..b77c5a907c118 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -21,8 +21,8 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to ``` $ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \ - CHANNEL path/to/rust/repo + CHANNEL VERSION ``` Remember to replace `CHANNEL` with the channel you produced dist artifacts of -and `path/to/rust/repo` with the path to your checkout of the Rust repository. +and `VERSION` with the current Rust version. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 7ee28cd6e5d95..cb04900c737e0 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -205,15 +205,20 @@ fn main() { // // Once the old release process is fully decommissioned, the environment variable, all the // related code in this tool and ./x.py dist hash-and-sign can be removed. - let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok(); - - // Avoid overloading the old server in legacy mode. - if legacy { - rayon::ThreadPoolBuilder::new() - .num_threads(1) - .build_global() - .expect("failed to initialize Rayon"); - } + let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some(); + + let num_threads = if legacy { + // Avoid overloading the old server in legacy mode. + 1 + } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") { + num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS") + } else { + num_cpus::get() + }; + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global() + .expect("failed to initialize Rayon"); let mut args = env::args().skip(1); let input = PathBuf::from(args.next().unwrap()); @@ -221,7 +226,6 @@ fn main() { let date = args.next().unwrap(); let s3_address = args.next().unwrap(); let channel = args.next().unwrap(); - let monorepo_path = args.next().unwrap(); // Do not ask for a passphrase while manually testing let mut passphrase = String::new(); @@ -231,7 +235,7 @@ fn main() { } Builder { - versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(), + versions: Versions::new(&channel, &input).unwrap(), input, output, diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 75b6979b54a78..79f2ef8dfc450 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Error}; +use anyhow::Error; use flate2::read::GzDecoder; use std::collections::HashMap; use std::fs::File; @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use tar::Archive; const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; +const RUSTC_VERSION: &str = include_str!("../../../version"); #[derive(Debug, Hash, Eq, PartialEq, Clone)] pub(crate) enum PkgType { @@ -87,26 +88,13 @@ pub(crate) struct VersionInfo { pub(crate) struct Versions { channel: String, - rustc_version: String, dist_path: PathBuf, versions: HashMap, } impl Versions { - pub(crate) fn new( - channel: &str, - dist_path: &Path, - monorepo_root: &Path, - ) -> Result { - Ok(Self { - channel: channel.into(), - rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version")) - .context("failed to read the rustc version from src/version")? - .trim() - .to_string(), - dist_path: dist_path.into(), - versions: HashMap::new(), - }) + pub(crate) fn new(channel: &str, dist_path: &Path) -> Result { + Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() }) } pub(crate) fn channel(&self) -> &str { @@ -184,10 +172,10 @@ impl Versions { ) -> Result { let component_name = package.tarball_component_name(); let version = match self.channel.as_str() { - "stable" => self.rustc_version.clone(), + "stable" => RUSTC_VERSION.into(), "beta" => "beta".into(), "nightly" => "nightly".into(), - _ => format!("{}-dev", self.rustc_version), + _ => format!("{}-dev", RUSTC_VERSION), }; if package.target_independent() { @@ -198,6 +186,6 @@ impl Versions { } pub(crate) fn rustc_version(&self) -> &str { - &self.rustc_version + RUSTC_VERSION } }