Rebuild stdlibs from source for aarch64-apple-darwin and wasm32-wasi

.gitignore

@@ -9,6 +9,7 @@
 /examples/cargo_manifest_dir/external_crate/bazel-*
 /examples/crate_universe/bazel-*
 /examples/crate_universe/*/bazel-*
+/examples/toolchain-to-rebuild-std/bazel-*
 /test/cc_common_link/bazel-*
 /test/cc_common_link/with_global_alloc/bazel-*
 /test/no_std/bazel-*

examples/toolchain-to-rebuild-std/.bazelrc

@@ -0,0 +1,10 @@
+# new feature
+build --@rules_rust//rust/settings:experimental_toolchain_generated_sysroot=True
+
+# enable toolchain debugging by default
+build --toolchain_resolution_debug='@rules_rust//rust:toolchain_type'
+
+# for toolchain debugging purposes
+build:beta --@rules_rust//rust/toolchain/channel:channel=beta
+build:stable --@rules_rust//rust/toolchain/channel:channel=stable
+build:nightly --@rules_rust//rust/toolchain/channel:channel=nightly

examples/toolchain-to-rebuild-std/BUILD.bazel

@@ -0,0 +1,15 @@
+platform(
+    name = "aarch64-apple-darwin",
+    constraint_values = [
+        "@platforms//os:osx",
+        "@platforms//cpu:aarch64",
+    ],
+)
+
+platform(
+    name = "wasm32-wasi",
+    constraint_values = [
+        "@platforms//os:wasi",
+        "@platforms//cpu:wasm32",
+    ],
+)

examples/toolchain-to-rebuild-std/README.md

@@ -0,0 +1,34 @@
+
+This directory demonstrates an example of how to rebuild the standard library
+for an arbitrary platform and custom configurations.
+
+TODO: Write up on how the toolchain bootrapping process works.
+
+To build an end-user rust_library target for specific target, run
+
+```
+bazel build //hello_lib --platforms=//:aarch64-apple-darwin
+```
+
+for `aarch64-apple-darwin` or
+
+```
+bazel build //hello_lib --platforms=//:wasm32-wasi
+```
+
+for `wasm32-wasi`.
+
+The toolchains orderly registered in WORKSPACE and _beta_channel_transition ensure
+that the standard library is rebuilt for the given platform.
+
+To only rebuild std, run
+
+```
+bazel build @stdlbs//:std --platforms=//:aarch64-apple-darwin
+```
+
+or
+
+```
+bazel build @stdlbs//:std --platforms=//:wasm32-wasi
+```

examples/toolchain-to-rebuild-std/WORKSPACE.bazel

@@ -0,0 +1,69 @@
+workspace(name = "rebuild_std_example")
+
+# Users of `rules_rust` will commonly be unable to load it
+# using a `local_repository`. Instead, to setup the rules,
+# please see https://bazelbuild.github.io/rules_rust/#setup
+local_repository(
+    name = "rules_rust",
+    path = "../..",
+)
+
+load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains")
+
+rules_rust_dependencies()
+
+register_toolchains(
+    "//dummy_cc_toolchain:dummy_cc_aarch64_toolchain",
+    "//dummy_cc_toolchain:dummy_cc_wasm32_toolchain",
+)
+
+# # The dates need to be sequential so that the stdlibs are built
+# # using the previous beta rustc
+[
+    previous_beta_date,
+    current_beta_date,
+] = [
+    "2023-08-22",
+    "2023-09-18",
+]
+
+rust_register_toolchains(
+    edition = "2021",
+    extra_rustc_flags = {
+        "aarch64-apple-darwin": ["--cap-lints=allow"],
+    },
+    extra_target_triples = [
+        "wasm32-wasi",
+        "aarch64-apple-darwin",
+    ],
+    versions = [
+        # beta toolchain to rebuild stdlibs
+        "beta/" + previous_beta_date,
+        # stable toolchain to build process_wrapper and tinyjson
+        "1.73.0",
+    ],
+)
+
+# http_archive(
+#     name = "wasi",
+#     build_file = "@rules_rust//util/import/3rdparty/crates:BUILD.wasi-0.11.0+wasi-snapshot-preview1.bazel",
+#     strip_prefix = "wasi-0.11.0",
+#     # sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+#     url = "https://github.com/bytecodealliance/wasi/archive/refs/tags/0.11.0.zip",
+# )
+
+load(":stdlibs.bzl", "stdlibs")
+
+# # Run
+# # bazel build @stage2_stdlibs//:std --@rules_rust//rust/toolchain/channel:channel=beta --platforms=//:aarch64-apple-darwin
+# # To rebuild standard library for aarch64-apple-darwin
+stdlibs(
+    name = "stdlibs",
+    beta_date = current_beta_date,
+    build_file = "//:stdlibs.BUILD.bazel",
+)
+
+register_toolchains(
+    "//toolchain:toolchain_aarch64-apple-darwin",
+    "//toolchain:toolchain_wasm32-wasi",
+)

examples/toolchain-to-rebuild-std/dummy_cc_toolchain/BUILD.bazel

@@ -0,0 +1,41 @@
+load("@rules_cc//cc:defs.bzl", "cc_toolchain")
+load(":dummy_cc_toolchain.bzl", "dummy_cc_config", "dummy_cc_toolchain")
+
+dummy_cc_toolchain(name = "dummy_cc_aarch64")
+
+toolchain(
+    name = "dummy_cc_aarch64_toolchain",
+    target_compatible_with = ["@platforms//cpu:aarch64"],
+    toolchain = ":dummy_cc_aarch64_toolchain_cc",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+toolchain(
+    name = "dummy_cc_wasm32_toolchain",
+    target_compatible_with = ["@platforms//cpu:wasm32"],
+    toolchain = ":dummy_cc_aarch64_toolchain_cc",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+cc_toolchain(
+    name = "dummy_cc_aarch64_toolchain_cc",
+    all_files = ":empty",
+    compiler_files = ":empty",
+    dwp_files = ":empty",
+    linker_files = ":empty",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    supports_param_files = 0,
+    toolchain_config = ":cc_toolchain_config",
+    toolchain_identifier = ":dummy_cc_aarch64",
+)
+
+dummy_cc_config(
+    name = "cc_toolchain_config",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+    visibility = ["//:__subpackages__"],
+)

examples/toolchain-to-rebuild-std/dummy_cc_toolchain/dummy_cc_toolchain.bzl

@@ -0,0 +1,30 @@
+"""Cc toolchain definitions for use on wasm platforms"""
+
+def _dummy_cc_toolchain_impl(_ctx):
+    # The `all_files` attribute is referenced by rustc_compile_action().
+    return [platform_common.ToolchainInfo(all_files = depset([]))]
+
+dummy_cc_toolchain = rule(
+    implementation = _dummy_cc_toolchain_impl,
+    attrs = {},
+)
+
+# dummy values from https://bazel.build/tutorials/ccp-toolchain-config#configuring_the_c_toolchain
+def _config_impl(ctx):
+    return cc_common.create_cc_toolchain_config_info(
+        ctx = ctx,
+        toolchain_identifier = "dummy-aarch64-cc-toolchain",
+        host_system_name = "unknown",
+        target_system_name = "unknown",
+        target_cpu = "unknown",
+        target_libc = "unknown",
+        compiler = "unknown",
+        abi_version = "unknown",
+        abi_libc_version = "unknown",
+    )
+
+dummy_cc_config = rule(
+    implementation = _config_impl,
+    attrs = {},
+    provides = [CcToolchainConfigInfo],
+)

examples/toolchain-to-rebuild-std/hello_program/BUILD.bazel

@@ -0,0 +1,28 @@
+load(
+    "@rules_rust//rust:defs.bzl",
+    "rust_binary",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+# This program uses std::os::wasi that is only available on WASI
+# When running `bazel build` with `--platforms=//:aarch64-apple-darwin`
+# rust toolchain is resolved to rebuild `std` for WASI specifically
+
+# As a counterexample, try removing `target_compatible_with` attr and rebuild
+# with `--platforms=//:aarch64-apple-darwin`. You'll see the following error
+#  --> hello_program/src/main_with_wasi.rs:4:14
+#   |
+# 4 | use std::os::wasi::prelude::*;
+#   |              ^^^^ could not find `wasi` in `os`
+# which is expected because std is now built for aarch64-apple-darwin.
+rust_binary(
+    name = "program_with_wasi",
+    srcs = [
+        "src/main_with_wasi.rs",
+    ],
+    target_compatible_with = select({
+        "@rules_rust//rust/platform:wasm32-wasi": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+)

examples/toolchain-to-rebuild-std/hello_program/src/main_with_wasi.rs

@@ -0,0 +1,13 @@
+// Copied from https://doc.rust-lang.org/std/os/wasi/index.html#examples
+
+use std::fs::File;
+use std::os::wasi::prelude::*;
+
+fn main() -> std::io::Result<()> {
+    let f = File::create("foo.txt")?;
+    let fd = f.as_raw_fd();
+
+    // use fd with native WASI bindings
+
+    Ok(())
+}