From 692bdb0a0689aeb0c162d8eae5c810a1d1f0c70c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Sun, 31 Oct 2021 14:49:58 +0300 Subject: [PATCH] Update Rust (#2761) This PR bumps rustc version and removes xargo. std dependencies are now built with cargo's new `-Zbuild-std` parameter. To work around rust-lang/wg-cargo-std-aware#23, we implement a tool "vendor-rust-std-deps" that reads `Cargo.lock` of a Rust toolchain std and manually vendors all the dependencies. These dependencies are then merged with the Rust RTS dependencies before the building the RTS in nix. RTS README updated with instructions to bump rustc. New rustc will enable more const functions, new API functions, stabilizations, and features for other RTS PRs. --- default.nix | 89 +++++-- nix/default.nix | 3 +- nix/xargo.nix | 31 --- rts/Makefile | 5 +- rts/README.md | 31 ++- rts/cargo-vendor-tools/Cargo.lock | 230 +++++++++++++++++ rts/cargo-vendor-tools/Cargo.toml | 15 ++ rts/cargo-vendor-tools/rustfmt.toml | 1 + .../src/vendor_rust_std_deps.rs | 235 ++++++++++++++++++ rts/motoko-rts-tests/Cargo.lock | 9 +- rts/motoko-rts-tests/src/main.rs | 2 +- rts/motoko-rts/Cargo.lock | 7 +- rts/motoko-rts/Cargo.toml | 14 -- rts/motoko-rts/Xargo.toml | 6 - rts/motoko-rts/native/Cargo.toml | 4 - rts/motoko-rts/src/lib.rs | 8 +- 16 files changed, 580 insertions(+), 110 deletions(-) delete mode 100644 nix/xargo.nix create mode 100644 rts/cargo-vendor-tools/Cargo.lock create mode 100644 rts/cargo-vendor-tools/Cargo.toml create mode 100644 rts/cargo-vendor-tools/rustfmt.toml create mode 100644 rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs delete mode 100644 rts/motoko-rts/Xargo.toml diff --git a/default.nix b/default.nix index edaf9775271..e6a595beea6 100644 --- a/default.nix +++ b/default.nix @@ -32,9 +32,11 @@ let llvmPackages_12.bintools rustc-nightly cargo-nightly - xargo wasmtime rust-bindgen + python3 + ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [ + libiconv ]; llvmEnv = '' @@ -152,46 +154,97 @@ in rec { rts = let - # We run this on motoko-rts-tests, to get the union of all - # dependencies - rustDeps = nixpkgs.rustPlatform-nightly.fetchCargoTarball { + # Build Rust package cargo-vendor-tools + cargoVendorTools = nixpkgs.rustPlatform.buildRustPackage rec { + name = "cargo-vendor-tools"; + src = ./rts/cargo-vendor-tools; + cargoSha256 = "0zi3fiq9sy6c9dv7fd2xc9lan85d16gfax47n6g6f5q5c1zb5r47"; + }; + + # Path to vendor-rust-std-deps, provided by cargo-vendor-tools + vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps"; + + # SHA256 of Rust std deps + rustStdDepsHash = "0wxx8prh66i19vd5078iky6x5bzs6ppz7c1vbcyx9h4fg0f7pfj6"; + + # Vendor directory for Rust std deps + rustStdDeps = nixpkgs.stdenvNoCC.mkDerivation { + name = "rustc-std-deps"; + + nativeBuildInputs = with nixpkgs; [ + curl + ]; + + buildCommand = '' + mkdir $out + cd $out + ${vendorRustStdDeps} ${nixpkgs.rustc-nightly} . + ''; + + outputHash = rustStdDepsHash; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }; + + # Vendor tarball of the RTS + rtsDeps = nixpkgs.rustPlatform.fetchCargoTarball { name = "motoko-rts-deps"; src = subpath ./rts; sourceRoot = "rts/motoko-rts-tests"; - sha256 = "0jyp3j8n5bj5cy1fd26d7h55zmc4v14qc2w8adxqwmsv5riqz41g"; + sha256 = "129gfmn96vm7di903pxirg7zybl83q6nkwiqr3rsy7l1q8667kxx"; copyLockfile = true; }; + + # Unpacked RTS deps + rtsDepsUnpacked = nixpkgs.stdenvNoCC.mkDerivation { + name = rtsDeps.name + "-unpacked"; + buildCommand = '' + tar xf ${rtsDeps} + mv *.tar.gz $out + ''; + }; + + # All dependencies needed to build the RTS, including Rust std deps, to + # allow `cargo -Zbuild-std`. (rust-lang/wg-cargo-std-aware#23) + allDeps = nixpkgs.stdenvNoCC.mkDerivation { + name = "merged-rust-deps"; + + buildCommand = '' + mkdir -p $out + cp -r ${rtsDepsUnpacked}/* $out/ + cp -r ${rustStdDeps}/* $out/ + ''; + }; in stdenv.mkDerivation { name = "moc-rts"; src = subpath ./rts; - nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo ]; + + nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo nixpkgs.cacert ]; buildInputs = rtsBuildInputs; preBuild = '' - export XARGO_HOME=$PWD/xargo-home export CARGO_HOME=$PWD/cargo-home - # this replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix + # This replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix mkdir -p $CARGO_HOME - echo "Using vendored sources from ${rustDeps}" - unpackFile ${rustDeps} + echo "Using vendored sources from ${rtsDeps}" + unpackFile ${allDeps} cat > $CARGO_HOME/config <<__END__ [source."crates-io"] "replace-with" = "vendored-sources" [source."vendored-sources"] - "directory" = "$(stripHash ${rustDeps})" + "directory" = "$(stripHash ${allDeps})" __END__ ${llvmEnv} export TOMMATHSRC=${nixpkgs.sources.libtommath} export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl export MUSL_WASI_SYSROOT=${musl-wasi-sysroot} - ''; doCheck = true; @@ -206,11 +259,16 @@ rec { cp mo-rts-debug.wasm $out/rts ''; - # This needs to be self-contained. Remove mention of - # nix path in debug message. + # This needs to be self-contained. Remove mention of nix path in debug + # message. preFixup = '' - remove-references-to -t ${nixpkgs.rustc-nightly} -t ${rustDeps} $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm + remove-references-to \ + -t ${nixpkgs.rustc-nightly} \ + -t ${rtsDeps} \ + -t ${rustStdDeps} \ + $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm ''; + allowedRequisites = []; }; @@ -440,7 +498,6 @@ rec { filecheck = nixpkgs.linkFarm "FileCheck" [ { name = "bin/FileCheck"; path = "${nixpkgs.llvm}/bin/FileCheck";} ]; - inherit (nixpkgs) xargo; # gitMinimal is used by nix/gitSource.nix; building it here warms the nix cache inherit (nixpkgs) gitMinimal; diff --git a/nix/default.nix b/nix/default.nix index 7e68f71465e..3a64fed5524 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -46,7 +46,7 @@ let # Rust nightly (self: super: let moz_overlay = import self.sources.nixpkgs-mozilla self super; - rust-channel = moz_overlay.rustChannelOf { date = "2020-07-22"; channel = "nightly"; }; + rust-channel = moz_overlay.rustChannelOf { date = "2021-10-25"; channel = "nightly"; }; in rec { rustc-nightly = rust-channel.rust.override { targets = [ @@ -62,7 +62,6 @@ let rustc = rustc-nightly; cargo = cargo-nightly; }; - xargo = self.callPackage ./xargo.nix {}; }) # wasm-profiler diff --git a/nix/xargo.nix b/nix/xargo.nix deleted file mode 100644 index c84995221b6..00000000000 --- a/nix/xargo.nix +++ /dev/null @@ -1,31 +0,0 @@ -# xargo is used to build motoko-rts for wasm32. We need to make a shared Wasm -# library for the RTS (that's what moc-ld supports) but Rust ships wasm32 -# libraries (core and std) without PIC relocation model, so we use xargo to make -# PIC versions of core and std. - -{ rustPlatform-nightly, fetchFromGitHub, lib }: - -rustPlatform-nightly.buildRustPackage rec { - name = "xargo"; - - src = fetchFromGitHub { - owner = "japaric"; - repo = "${name}"; - rev = "16035a7c401262824edcb87e1401fe4b05a5ccc0"; - sha256 = "0m1dg7vwmmlpqp20p219gsm7zbnnii6lik6hc2vvfsdmnygf271l"; - fetchSubmodules = true; - }; - - cargoSha256 = "171a7xm47qdcdd6n67plyvnsxp00hn6skx5vzxd8a2kmblfqn5gy"; - - doCheck = false; - USER = "nobody"; # for xargo tests (if we would run them) - - meta = with lib; { - description = "The sysroot manager that lets you build and customize std"; - homepage = "https://github.com/japaric/xargo"; - license = licenses.mit; - maintainers = []; - platforms = platforms.unix; - }; -} diff --git a/rts/Makefile b/rts/Makefile index 415881dc19a..eb5fd2efac1 100644 --- a/rts/Makefile +++ b/rts/Makefile @@ -223,11 +223,11 @@ $(TOMMATH_BINDINGS_RS): | _build $(RTS_RUST_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm - cd motoko-rts && xargo build --release --target=wasm32-unknown-emscripten + cd motoko-rts && cargo build --release --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc cp motoko-rts/target/wasm32-unknown-emscripten/release/libmotoko_rts.a $@ $(RTS_RUST_DEBUG_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm - cd motoko-rts && xargo build --target=wasm32-unknown-emscripten + cd motoko-rts && cargo build --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc cp motoko-rts/target/wasm32-unknown-emscripten/debug/libmotoko_rts.a $@ # @@ -286,5 +286,4 @@ clean: mo-rts-debug.wasm \ motoko-rts/target \ motoko-rts-tests/target \ - motoko-rts/xargo-home \ motoko-rts/cargo-home diff --git a/rts/README.md b/rts/README.md index 3e464e8b6fd..d1ba8ffc1df 100644 --- a/rts/README.md +++ b/rts/README.md @@ -58,31 +58,36 @@ See `motoko-rts/src/bigint.rs` for the technical details. Rust build ---------- -The Rust parts are built from `motoko-rts`, using `xargo` and `cargo`. +To build Motoko RTS in nix we need pre-fetch Rust dependencies. This works in +`nix-build` by: -To build this in nix, we need pre-fetch some dependencies (currently -`compiler_builtins` and `libc`). This works in `nix-build` by: - - * Adding`compiler_builtins` as a dependency in `Cargo.toml` (even though not - needed), so that it shows up with a hash in `Cargo.lock` - - * Building a directory with vendored sources in `default.nix` (see `rustDeps`) + * Building a directory with vendored sources in `default.nix` * Configuring `cargo` to use that vendored directory (see `preBuild`) If you change dependencies (e.g. bump versions, add more crates), - 1. Add them to `Cargo.toml` - 2. Make sure that `Cargo.lock` is up to date - 3. In `default.nix`, invalidate the `sha256` of `rustDeps` (e.g. change one + 1. Make sure that `motoko-rts-tests/Cargo.lock` is up to date. This can be + done by running `cargo build --target=wasm32-wasi` in `motoko-rts-tests/` + directory. + 2. In `default.nix`, invalidate the `sha256` of `rtsDeps` (e.g. change one character) - 4. Run `nix-build -A rts`. You should get an error message about the actual + 3. Run `nix-build -A rts`. You should get an error message about the actual checksum. - 5. Set that as `sha256` of `rustDeps` in `default.nix` + 4. Set that as `sha256` of `rtsDeps` in `default.nix` Warning: nix will happily use a stale version of the dependencies if you do not do step 3. +**Updating rustc**: + +1. Update Rust version in `nix/default.nix`, in the line with + `moz_overlay.rustChannelOf { ... }`. +2. Invalidate `rustStdDepsHash` in `default.nix`. +3. Run `nix-build -A rts`. You should get an error message about the expected + value of `rustStdDepsHash`. +4. Update `rustStdDepsHash` with the expected value in the error message. + Running RTS tests ----------------- diff --git a/rts/cargo-vendor-tools/Cargo.lock b/rts/cargo-vendor-tools/Cargo.lock new file mode 100644 index 00000000000..e372c4c585a --- /dev/null +++ b/rts/cargo-vendor-tools/Cargo.lock @@ -0,0 +1,230 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cargo-vendor-tools" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", + "sha2", + "toml", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "libc" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" + +[[package]] +name = "serde_json" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +dependencies = [ + "block-buffer", + "cfg-if", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/rts/cargo-vendor-tools/Cargo.toml b/rts/cargo-vendor-tools/Cargo.toml new file mode 100644 index 00000000000..7b63e2002cc --- /dev/null +++ b/rts/cargo-vendor-tools/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cargo-vendor-tools" +version = "0.1.0" +authors = ["dfinity "] +edition = "2018" + +[[bin]] +name = "vendor-rust-std-deps" +path = "src/vendor_rust_std_deps.rs" + +[dependencies] +clap = "2.33.3" +serde_json = "1.0.64" +sha2 = "0.9.5" +toml = "0.5.8" diff --git a/rts/cargo-vendor-tools/rustfmt.toml b/rts/cargo-vendor-tools/rustfmt.toml new file mode 100644 index 00000000000..66aa257537b --- /dev/null +++ b/rts/cargo-vendor-tools/rustfmt.toml @@ -0,0 +1 @@ +struct_lit_width = 40 diff --git a/rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs b/rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs new file mode 100644 index 00000000000..3c3bf7e7848 --- /dev/null +++ b/rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs @@ -0,0 +1,235 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use clap::{App, Arg}; +use serde_json as json; +use sha2::{Digest, Sha256}; + +static DESCR: &str = " +Given a Rust toolchain path and a vendor directory, this program vendors Rust std dependencies in +the given vendor directory. This directory can then be used to build packages with `-Zbuild-std` +parameters in nix. +"; + +fn main() { + let args = App::new("vendor-rust-std-deps") + .long_about(DESCR) + .arg( + Arg::with_name("rust_install_path") + .value_name("RUST_INSTALL_PATH") + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name("vendor_dir") + .value_name("VENDOR_DIR") + .takes_value(true) + .required(true), + ) + .get_matches(); + + let rust_install_path = args.value_of("rust_install_path").unwrap(); + let vendor_dir = args.value_of("vendor_dir").unwrap(); + + let compiler_cargo_lock_path = format!( + "{}/{}", + rust_install_path, "lib/rustlib/src/rust/Cargo.lock" + ); + + println!("Reading compiler Cargo.lock ..."); + let compiler_deps = cargo_lock_deps(&compiler_cargo_lock_path); + + println!( + "Fetching and unpacking compiler dependencies in {} ...", + vendor_dir + ); + + let total_deps = compiler_deps.len(); + for (dep_idx, Dep { url, name, tarball_checksum }) in compiler_deps.into_iter().enumerate() { + println!(" {} ({}/{})", name, dep_idx + 1, total_deps); + + // + // Fetch the package tarball + // + + // curl --location + let Output { status, stdout: curl_stdout, stderr } = Command::new("curl") + .args(&["--location", &url, "--insecure"]) + .output() + .unwrap(); + + if !status.success() { + panic!( + "`curl --location {}` returned {:?}:\n{}", + url, + status.code(), + String::from_utf8_lossy(&stderr) + ); + } + + let package_vendor_dir = format!("{}/{}", vendor_dir, name); + + // + // Create package vendor dir + // + + std::fs::create_dir_all(&package_vendor_dir).unwrap(); + + // + // Unpack package + // + + let mut tar_cmd = Command::new("tar") + .args(&["xz"]) + .current_dir(&vendor_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let mut tar_stdin = tar_cmd.stdin.take().unwrap(); + tar_stdin.write_all(&curl_stdout).unwrap(); + drop(tar_stdin); + + let Output { status, stdout: _, stderr } = tar_cmd.wait_with_output().unwrap(); + + if !status.success() { + panic!( + "`tar xz` returned {:?}:\n{}", + status.code(), + String::from_utf8_lossy(&stderr) + ); + } + + // + // Generate file checksums + // + + let mut file_checksums: json::map::Map = Default::default(); + generate_file_checksums( + &package_vendor_dir, + PathBuf::from(&package_vendor_dir).as_ref(), + &mut file_checksums, + ); + + let mut json_map = json::map::Map::new(); + json_map.insert("files".to_owned(), json::Value::Object(file_checksums)); + json_map.insert( + "package".to_owned(), + json::Value::String(tarball_checksum.to_owned()), + ); + + let mut checksum_file = + std::fs::File::create(format!("{}/.cargo-checksum.json", package_vendor_dir)).unwrap(); + json::ser::to_writer_pretty(&mut checksum_file, &json_map).unwrap(); + } +} + +#[derive(Debug)] +struct Dep { + /// URL to fetch the package tarball + url: String, + + /// - + name: String, + + /// Expected checksum (SHA256) of the tarball + tarball_checksum: String, +} + +fn cargo_lock_deps(lock_file_path: &str) -> Vec { + let mut deps = vec![]; + + let lock_file = std::fs::read_to_string(lock_file_path).unwrap(); + + let toml_value: toml::value::Value = toml::de::from_str(&lock_file).unwrap(); + + let packages = toml_value.as_table().unwrap().get("package").unwrap(); + + for package in packages.as_array().unwrap() { + let package_tbl = package.as_table().unwrap(); + if let Some(source) = package_tbl.get("source") { + if source.as_str().unwrap() == "registry+https://github.com/rust-lang/crates.io-index" { + let name = package_tbl.get("name").unwrap().as_str().unwrap(); + let version = package_tbl.get("version").unwrap().as_str().unwrap(); + let checksum = package_tbl.get("checksum").unwrap().as_str().unwrap(); + deps.push(Dep { + url: format!( + "https://crates.io/api/v1/crates/{}/{}/download", + name, version + ), + name: format!("{}-{}", name, version), + tarball_checksum: checksum.to_owned(), + }); + } + } + } + + deps +} + +static IGNORED_FILES: [&str; 7] = [ + ".", + "..", + ".gitattributes", + ".gitignore", + ".cargo-ok", + ".cargo-checksum.json", + ".cargo_vcs_info.json", +]; + +fn generate_file_checksums( + root: &str, + dir: &Path, + file_checksums: &mut json::map::Map, +) { + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let file_name = entry.file_name(); + + if IGNORED_FILES + .iter() + .any(|ignored_file| *ignored_file == file_name) + { + continue; + } + + match entry.file_type() { + Err(err) => { + eprintln!( + "Error while getting type of file {:?}: {:?}", + file_name, err + ); + } + Ok(file_type) => { + if file_type.is_dir() { + let mut dir_path = PathBuf::new(); + dir_path.push(dir); + dir_path.push(entry.file_name()); + generate_file_checksums(root, &dir_path, file_checksums); + } else { + let mut file_path = PathBuf::new(); + file_path.push(dir); + file_path.push(entry.file_name()); + + let rel_file_path = file_path.strip_prefix(root).unwrap(); + + let file_contents = std::fs::read(&file_path).unwrap(); + + let mut hasher = Sha256::new(); + hasher.update(&file_contents); + let hash = hasher.finalize(); + let hash_str = format!("{:x}", hash); + + let old = file_checksums.insert( + rel_file_path.to_string_lossy().to_string(), + json::Value::String(hash_str), + ); + assert!(old.is_none()); + } + } + } + } +} diff --git a/rts/motoko-rts-tests/Cargo.lock b/rts/motoko-rts-tests/Cargo.lock index 6b60a63f4e4..834253a1868 100644 --- a/rts/motoko-rts-tests/Cargo.lock +++ b/rts/motoko-rts-tests/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "autocfg" version = "1.0.1" @@ -24,12 +26,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "compiler_builtins" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b" - [[package]] name = "fxhash" version = "0.2.1" @@ -66,7 +62,6 @@ checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" name = "motoko-rts" version = "0.1.0" dependencies = [ - "compiler_builtins", "libc", "motoko-rts-macros", "static_assertions", diff --git a/rts/motoko-rts-tests/src/main.rs b/rts/motoko-rts-tests/src/main.rs index f18df6ac569..9f77525e034 100644 --- a/rts/motoko-rts-tests/src/main.rs +++ b/rts/motoko-rts-tests/src/main.rs @@ -1,4 +1,4 @@ -#![feature(ptr_offset_from, map_first_last, clamp)] +#![feature(map_first_last)] mod bigint; mod bitmap; diff --git a/rts/motoko-rts/Cargo.lock b/rts/motoko-rts/Cargo.lock index 1e4ffb001c7..dc5205045cf 100644 --- a/rts/motoko-rts/Cargo.lock +++ b/rts/motoko-rts/Cargo.lock @@ -1,10 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "compiler_builtins" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b" +version = 3 [[package]] name = "libc" @@ -16,7 +12,6 @@ checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" name = "motoko-rts" version = "0.1.0" dependencies = [ - "compiler_builtins", "libc", "motoko-rts-macros", "static_assertions", diff --git a/rts/motoko-rts/Cargo.toml b/rts/motoko-rts/Cargo.toml index 575b327b100..b7eaa74f564 100644 --- a/rts/motoko-rts/Cargo.toml +++ b/rts/motoko-rts/Cargo.toml @@ -23,20 +23,6 @@ libc = { version = "0.2.81", default_features = false } motoko-rts-macros = { path = "../motoko-rts-macros" } static_assertions = "1.1.0" -# Added here so that it ends up in Cargo.lock, so that nix will pre-fetch it -[dependencies.compiler_builtins] -version = "0.1.39" -# Without this feature we get dozens of duplicate symbol errors when generating -# the final shared .wasm file: -# -# wasm-ld: error: duplicate symbol: __multi3 -# >>> defined in _build/wasm/libmotoko_rts.a(compiler_builtins-d709bd899857aa61.compiler_builtins.3abndchk-cgu.0.rcgu.o) -# >>> defined in _build/wasm/libmotoko_rts.a(compiler_builtins-06d1ead628e1f468.compiler_builtins.6moz1ltd-cgu.0.rcgu.o) -# -# It seems like we're linking multiple versions of compiler_builtins in the same -# shared library, which we should fix at some point. TODO -features = ["mangled-names"] - [profile.dev] panic = "abort" diff --git a/rts/motoko-rts/Xargo.toml b/rts/motoko-rts/Xargo.toml deleted file mode 100644 index d3910abede1..00000000000 --- a/rts/motoko-rts/Xargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[dependencies.core] -stage = 0 - -[dependencies.compiler_builtins] -stage = 1 -version = "0.1.39" diff --git a/rts/motoko-rts/native/Cargo.toml b/rts/motoko-rts/native/Cargo.toml index 9c8913e32ff..5e22f655e31 100644 --- a/rts/motoko-rts/native/Cargo.toml +++ b/rts/motoko-rts/native/Cargo.toml @@ -13,10 +13,6 @@ libc = { version = "0.2.73", default_features = false } motoko-rts-macros = { path = "../../motoko-rts-macros" } static_assertions = "1.1.0" -[dependencies.compiler_builtins] -version = "0.1.39" -features = ["mangled-names"] - [profile.dev] panic = "abort" diff --git a/rts/motoko-rts/src/lib.rs b/rts/motoko-rts/src/lib.rs index df68ab06b74..e036b0d3096 100644 --- a/rts/motoko-rts/src/lib.rs +++ b/rts/motoko-rts/src/lib.rs @@ -2,13 +2,7 @@ #![no_std] // TODO (osa): Some of these are stabilized, we need to update rustc -#![feature( - arbitrary_self_types, - assoc_char_funcs, - core_intrinsics, - panic_info_message, - ptr_offset_from -)] +#![feature(arbitrary_self_types, core_intrinsics, panic_info_message)] #[macro_use] mod print;