Update Rust (#2761)

This PR bumps rustc version and removes xargo. std dependencies are now built
with cargo's new `-Zbuild-std` parameter.

To work around rust-lang/wg-cargo-std-aware#23, we implement a tool
"vendor-rust-std-deps" that reads `Cargo.lock` of a Rust toolchain std and
manually vendors all the dependencies.

These dependencies are then merged with the Rust RTS dependencies before the
building the RTS in nix.

RTS README updated with instructions to bump rustc.

New rustc will enable more const functions, new API functions, stabilizations,
and features for other RTS PRs.

Author: osa1

default.nix

@@ -32,9 +32,11 @@ let
     llvmPackages_12.bintools
     rustc-nightly
     cargo-nightly
-    xargo
     wasmtime
     rust-bindgen
+    python3
+  ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [
+    libiconv
   ];
 
   llvmEnv = ''
@@ -152,46 +154,97 @@ in
 rec {
   rts =
     let
-      # We run this on motoko-rts-tests, to get the union of all
-      # dependencies
-      rustDeps = nixpkgs.rustPlatform-nightly.fetchCargoTarball {
+      # Build Rust package cargo-vendor-tools
+      cargoVendorTools = nixpkgs.rustPlatform.buildRustPackage rec {
+        name = "cargo-vendor-tools";
+        src = ./rts/cargo-vendor-tools;
+        cargoSha256 = "0zi3fiq9sy6c9dv7fd2xc9lan85d16gfax47n6g6f5q5c1zb5r47";
+      };
+
+      # Path to vendor-rust-std-deps, provided by cargo-vendor-tools
+      vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps";
+
+      # SHA256 of Rust std deps
+      rustStdDepsHash = "0wxx8prh66i19vd5078iky6x5bzs6ppz7c1vbcyx9h4fg0f7pfj6";
+
+      # Vendor directory for Rust std deps
+      rustStdDeps = nixpkgs.stdenvNoCC.mkDerivation {
+        name = "rustc-std-deps";
+
+        nativeBuildInputs = with nixpkgs; [
+          curl
+        ];
+
+        buildCommand = ''
+          mkdir $out
+          cd $out
+          ${vendorRustStdDeps} ${nixpkgs.rustc-nightly} .
+        '';
+
+        outputHash = rustStdDepsHash;
+        outputHashAlgo = "sha256";
+        outputHashMode = "recursive";
+      };
+
+      # Vendor tarball of the RTS
+      rtsDeps = nixpkgs.rustPlatform.fetchCargoTarball {
         name = "motoko-rts-deps";
         src = subpath ./rts;
         sourceRoot = "rts/motoko-rts-tests";
-        sha256 = "0jyp3j8n5bj5cy1fd26d7h55zmc4v14qc2w8adxqwmsv5riqz41g";
+        sha256 = "129gfmn96vm7di903pxirg7zybl83q6nkwiqr3rsy7l1q8667kxx";
         copyLockfile = true;
       };
+
+      # Unpacked RTS deps
+      rtsDepsUnpacked = nixpkgs.stdenvNoCC.mkDerivation {
+        name = rtsDeps.name + "-unpacked";
+        buildCommand = ''
+          tar xf ${rtsDeps}
+          mv *.tar.gz $out
+        '';
+      };
+
+      # All dependencies needed to build the RTS, including Rust std deps, to
+      # allow `cargo -Zbuild-std`. (rust-lang/wg-cargo-std-aware#23)
+      allDeps = nixpkgs.stdenvNoCC.mkDerivation {
+          name = "merged-rust-deps";
+
+          buildCommand = ''
+            mkdir -p $out
+            cp -r ${rtsDepsUnpacked}/* $out/
+            cp -r ${rustStdDeps}/* $out/
+          '';
+        };
     in
 
     stdenv.mkDerivation {
       name = "moc-rts";
 
       src = subpath ./rts;
-      nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo ];
+
+      nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo nixpkgs.cacert ];
 
       buildInputs = rtsBuildInputs;
 
       preBuild = ''
-        export XARGO_HOME=$PWD/xargo-home
         export CARGO_HOME=$PWD/cargo-home
 
-        # this replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix
+        # This replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix
         mkdir -p $CARGO_HOME
-        echo "Using vendored sources from ${rustDeps}"
-        unpackFile ${rustDeps}
+        echo "Using vendored sources from ${rtsDeps}"
+        unpackFile ${allDeps}
         cat > $CARGO_HOME/config <<__END__
           [source."crates-io"]
           "replace-with" = "vendored-sources"
 
           [source."vendored-sources"]
-          "directory" = "$(stripHash ${rustDeps})"
+          "directory" = "$(stripHash ${allDeps})"
         __END__
 
         ${llvmEnv}
         export TOMMATHSRC=${nixpkgs.sources.libtommath}
         export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl
         export MUSL_WASI_SYSROOT=${musl-wasi-sysroot}
-
       '';
 
       doCheck = true;
@@ -206,11 +259,16 @@ rec {
         cp mo-rts-debug.wasm $out/rts
       '';
 
-      # This needs to be self-contained. Remove mention of
-      # nix path in debug message.
+      # This needs to be self-contained. Remove mention of nix path in debug
+      # message.
       preFixup = ''
-        remove-references-to -t ${nixpkgs.rustc-nightly} -t ${rustDeps} $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm
+        remove-references-to \
+          -t ${nixpkgs.rustc-nightly} \
+          -t ${rtsDeps} \
+          -t ${rustStdDeps} \
+          $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm
       '';
+
       allowedRequisites = [];
     };
 
@@ -440,7 +498,6 @@ rec {
 
   filecheck = nixpkgs.linkFarm "FileCheck"
     [ { name = "bin/FileCheck"; path = "${nixpkgs.llvm}/bin/FileCheck";} ];
-  inherit (nixpkgs) xargo;
 
   # gitMinimal is used by nix/gitSource.nix; building it here warms the nix cache
   inherit (nixpkgs) gitMinimal;

nix/default.nix

@@ -46,7 +46,7 @@ let
         # Rust nightly
         (self: super: let
           moz_overlay = import self.sources.nixpkgs-mozilla self super;
-          rust-channel = moz_overlay.rustChannelOf { date = "2020-07-22"; channel = "nightly"; };
+          rust-channel = moz_overlay.rustChannelOf { date = "2021-10-25"; channel = "nightly"; };
         in rec {
           rustc-nightly = rust-channel.rust.override {
             targets = [
@@ -62,7 +62,6 @@ let
             rustc = rustc-nightly;
             cargo = cargo-nightly;
           };
-          xargo = self.callPackage ./xargo.nix {};
         })
 
         # wasm-profiler

nix/xargo.nix

@@ -223,11 +223,11 @@ $(TOMMATH_BINDINGS_RS): | _build
 
 
 $(RTS_RUST_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
-	cd motoko-rts && xargo build --release --target=wasm32-unknown-emscripten
+	cd motoko-rts && cargo build --release --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
 	cp motoko-rts/target/wasm32-unknown-emscripten/release/libmotoko_rts.a $@
 
 $(RTS_RUST_DEBUG_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
-	cd motoko-rts && xargo build --target=wasm32-unknown-emscripten
+	cd motoko-rts && cargo build --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
 	cp motoko-rts/target/wasm32-unknown-emscripten/debug/libmotoko_rts.a $@
 
 #
@@ -286,5 +286,4 @@ clean:
 	  mo-rts-debug.wasm \
 	  motoko-rts/target \
 	  motoko-rts-tests/target \
-	  motoko-rts/xargo-home \
 	  motoko-rts/cargo-home

rts/Makefile

@@ -58,31 +58,36 @@ See `motoko-rts/src/bigint.rs` for the technical details.
 Rust build
 ----------
 
-The Rust parts are built from `motoko-rts`, using `xargo` and `cargo`.
+To build Motoko RTS in nix we need pre-fetch Rust dependencies. This works in
+`nix-build` by:
 
-To build this in nix, we need pre-fetch some dependencies (currently
-`compiler_builtins` and `libc`). This works in `nix-build` by:
-
- * Adding`compiler_builtins` as a dependency in `Cargo.toml` (even though not
-   needed), so that it shows up with a hash in `Cargo.lock`
-
- * Building a directory with vendored sources in `default.nix` (see `rustDeps`)
+ * Building a directory with vendored sources in `default.nix`
 
  * Configuring `cargo` to use that vendored directory (see `preBuild`)
 
 If you change dependencies (e.g. bump versions, add more crates),
 
- 1. Add them to `Cargo.toml`
- 2. Make sure that `Cargo.lock` is up to date
- 3. In `default.nix`, invalidate the `sha256` of `rustDeps` (e.g. change one
+ 1. Make sure that `motoko-rts-tests/Cargo.lock` is up to date. This can be
+    done by running `cargo build --target=wasm32-wasi` in `motoko-rts-tests/`
+    directory.
+ 2. In `default.nix`, invalidate the `sha256` of `rtsDeps` (e.g. change one
     character)
- 4. Run `nix-build -A rts`. You should get an error message about the actual
+ 3. Run `nix-build -A rts`. You should get an error message about the actual
     checksum.
- 5. Set that as `sha256` of `rustDeps` in `default.nix`
+ 4. Set that as `sha256` of `rtsDeps` in `default.nix`
 
 Warning: nix will happily use a stale version of the dependencies if you do not
 do step 3.
 
+**Updating rustc**:
+
+1. Update Rust version in `nix/default.nix`, in the line with
+   `moz_overlay.rustChannelOf { ... }`.
+2. Invalidate `rustStdDepsHash` in `default.nix`.
+3. Run `nix-build -A rts`. You should get an error message about the expected
+   value of `rustStdDepsHash`.
+4. Update `rustStdDepsHash` with the expected value in the error message.
+
 Running RTS tests
 -----------------