From fd587513d97c38b04721df93efd2ec4272d7f712 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Fri, 9 Aug 2024 11:47:37 -0400 Subject: [PATCH 01/11] Update Rust version and patches --- config.sh | 2 +- patches/rust.patch | 99 ++++------------------------------------------ 2 files changed, 9 insertions(+), 92 deletions(-) diff --git a/config.sh b/config.sh index 494ebd4..cca11de 100755 --- a/config.sh +++ b/config.sh @@ -2,7 +2,7 @@ param1=${1:-} -export RUST_COMMIT=9b00956e56009bab2aa15d7bff10916599e3d6d6 +export RUST_COMMIT=051478957371ee0084a7c0913941d2a8c4757bb9 # If -rust_commit specified then only export the rust commit variable if [ "$param1" == "-rust_commit" ]; then diff --git a/patches/rust.patch b/patches/rust.patch index 9dd7242..7ab8a14 100644 --- a/patches/rust.patch +++ b/patches/rust.patch @@ -1,36 +1,8 @@ -diff --git a/Cargo.lock b/Cargo.lock -index 927e93f2..dfbe7db5 100644 ---- a/Cargo.lock -+++ b/Cargo.lock -@@ -734,9 +734,8 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335" - - [[package]] - name = "compiler_builtins" --version = "0.1.108" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "d68bc55329711cd719c2687bb147bc06211b0521f97ef398280108ccb23227e9" -+version = "0.1.105" -+source = "git+https://github.com/rust-lang/compiler-builtins?rev=0.1.105#80ed5f8c3ddbea7986c4a8243e42b963cb954b42" - dependencies = [ - "cc", - "rustc-std-workspace-core", -diff --git a/Cargo.toml b/Cargo.toml -index 5dd315ef2f7..680fb821c4c 100644 ---- a/Cargo.toml -+++ b/Cargo.toml -@@ -120,6 +120,7 @@ strip = true - rustc-std-workspace-core = { path = 'library/rustc-std-workspace-core' } - rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' } - rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' } -+compiler_builtins = { git = "https://github.com/rust-lang/compiler-builtins", rev = "0.1.105" } - - [patch."https://github.com/rust-lang/rust-clippy"] - clippy_lints = { path = "src/tools/clippy/clippy_lints" } diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs -index 941d767b..1b4004c4 100644 +index fe07d1167..07e786b6c 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs -@@ -1618,6 +1618,7 @@ fn $module() { +@@ -1731,6 +1731,7 @@ fn $module() { ("x86_64-unikraft-linux-musl", x86_64_unikraft_linux_musl), @@ -39,25 +11,25 @@ index 941d767b..1b4004c4 100644 ("riscv32im-risc0-zkvm-elf", riscv32im_risc0_zkvm_elf), ("riscv32im-unknown-none-elf", riscv32im_unknown_none_elf), diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs -index 5ed6b357e20..a31645ab6e7 100644 +index cde090637..07c9c5758 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs -@@ -91,6 +91,10 @@ +@@ -95,6 +95,10 @@ /* Extra values not defined in the built-in targets yet, but used in std */ (Some(Mode::Std), "target_env", Some(&["libnx", "p2"])), - // (Some(Mode::Std), "target_os", Some(&[])), + (Some(Mode::Std), "target_os", Some(&["visionos"])), + // #[cfg(bootstrap)] zkvm + (Some(Mode::Std), "target_os", Some(&["zkvm"])), + // #[cfg(bootstrap)] athena + (Some(Mode::Std), "target_vendor", Some(&["athena", "succinct"])), (Some(Mode::Std), "target_arch", Some(&["arm64ec", "spirv", "nvptx", "xtensa"])), + (Some(Mode::ToolStd), "target_os", Some(&["visionos"])), /* Extra names used by dependencies */ - // FIXME: Used by serde_json, but we should not be triggering on external dependencies. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs -index eab9138b..60507063 100644 +index bed76263b..121843a30 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs -@@ -123,6 +123,7 @@ +@@ -125,6 +125,7 @@ "powerpc-unknown-linux-gnu", "powerpc64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", @@ -115,58 +87,3 @@ index 00000000..a84f76c2 + } +} + -diff --git a/library/std/src/sys/pal/zkvm/args.rs b/library/std/src/sys/pal/zkvm/args.rs -index 7753cf63840..583c16e3a47 100644 ---- a/library/std/src/sys/pal/zkvm/args.rs -+++ b/library/std/src/sys/pal/zkvm/args.rs -@@ -1,6 +1,7 @@ - use super::{abi, WORD_SIZE}; - use crate::ffi::OsString; - use crate::fmt; -+use crate::sys::os_str; - use crate::sys_common::FromInner; - - pub struct Args { -@@ -33,7 +34,7 @@ fn argv(i: usize) -> OsString { - // "os_str". - let arg_bytes: &[u8] = - unsafe { crate::slice::from_raw_parts(words.cast() as *const u8, arg_len) }; -- OsString::from_inner(super::os_str::Buf { inner: arg_bytes.to_vec() }) -+ OsString::from_inner(os_str::Buf { inner: arg_bytes.to_vec() }) - } - } - -diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs -index 228a976dbab..c5a1843d5ff 100644 ---- a/library/std/src/sys/pal/zkvm/mod.rs -+++ b/library/std/src/sys/pal/zkvm/mod.rs -@@ -12,8 +12,6 @@ - pub mod alloc; - #[path = "../zkvm/args.rs"] - pub mod args; --#[path = "../unix/cmath.rs"] --pub mod cmath; - pub mod env; - #[path = "../unsupported/fs.rs"] - pub mod fs; -diff --git a/library/std/src/sys/pal/zkvm/os.rs b/library/std/src/sys/pal/zkvm/os.rs -index d8739ee3824..759beb2d306 100644 ---- a/library/std/src/sys/pal/zkvm/os.rs -+++ b/library/std/src/sys/pal/zkvm/os.rs -@@ -5,6 +5,7 @@ - use crate::io; - use crate::marker::PhantomData; - use crate::path::{self, PathBuf}; -+use crate::sys::os_str; - use crate::sys_common::FromInner; - - pub fn errno() -> i32 { -@@ -111,7 +112,7 @@ pub fn getenv(varname: &OsStr) -> Option { - // reimplement "os_str" instead of just using the generic unix - // "os_str". - let u8s: &[u8] = unsafe { crate::slice::from_raw_parts(words.cast() as *const u8, nbytes) }; -- Some(OsString::from_inner(super::os_str::Buf { inner: u8s.to_vec() })) -+ Some(OsString::from_inner(os_str::Buf { inner: u8s.to_vec() })) - } - - pub fn setenv(_: &OsStr, _: &OsStr) -> io::Result<()> { From a0f117896cec2baf038ebc74c2ebc3ed5cedde47 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Mon, 12 Aug 2024 14:40:13 -0700 Subject: [PATCH 02/11] Update config.toml Update change-id --- patches/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/config.toml b/patches/config.toml index 650af04..6a085cf 100644 --- a/patches/config.toml +++ b/patches/config.toml @@ -1,4 +1,4 @@ -change-id = 121754 +change-id = 125535 [build] target = ["riscv32em-athena-zkvm-elf"] From de242ac1370d108493152e6178730b0e483187cd Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Mon, 12 Aug 2024 14:40:28 -0700 Subject: [PATCH 03/11] Update config.ci.toml Update change-id --- patches/config.ci.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/config.ci.toml b/patches/config.ci.toml index 9f1ff9c..afd4360 100644 --- a/patches/config.ci.toml +++ b/patches/config.ci.toml @@ -1,4 +1,4 @@ -change-id = 121754 +change-id = 125535 [build] target = ["riscv32em-athena-zkvm-elf"] From 9dce500bfae3b50eee09eb14d1274b37aa03778b Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Mon, 12 Aug 2024 19:38:19 -0700 Subject: [PATCH 04/11] Compilation is working again Include dependency on riscv toolchain Include small llvm patch --- build.sh | 11 ++++++++++- patch.sh | 3 +++ patches/config.ci.toml | 4 ++++ patches/config.toml | 4 ++++ patches/llvm.patch | 17 +++++++++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 patches/llvm.patch diff --git a/build.sh b/build.sh index e46439b..5f392b3 100755 --- a/build.sh +++ b/build.sh @@ -4,6 +4,15 @@ set -euo pipefail source config.sh +# Rust requires a custom target file to exist for our custom target as part of the bootstrap build, +# but it doesn't actually look at the contents. +touch /tmp/riscv32em-athena-zkvm-elf.json + # Will create component archives (dists) ./rust/build/dist cd rust -CARGO_TARGET_RISCV32EM_ATHENA_ZKVM_ELF_RUSTFLAGS="-Cpasses=loweratomic" ./x build --stage 2 +#RUSTFLAGS="-C link-arg=-L$HOME/compiler_rt_build_riscv32em-athena-zkvm-elf/lib/baremetal" \ +#LIBRARY_PATH="$HOME/compiler_rt_build_riscv32em-athena-zkvm-elf/lib/baremetal" \ +RUST_TARGET_PATH="/tmp" \ +CARGO_TARGET_RISCV32EM_ATHENA_ZKVM_ELF_RUSTFLAGS="-Cpasses=loweratomic" \ +CFLAGS_riscv32em_athena_zkvm_elf="-ffunction-sections -fdata-sections -fPIC -march=rv32em -mabi=ilp32e" \ +./x build --stage 2 diff --git a/patch.sh b/patch.sh index 2ec3ec9..0963cab 100755 --- a/patch.sh +++ b/patch.sh @@ -5,3 +5,6 @@ set -euo pipefail cd rust patch -p1 < ../patches/rust.patch cp ../patches/config.toml ./ + +cd src/llvm-project +patch -p1 < ../../../patches/llvm.patch diff --git a/patches/config.ci.toml b/patches/config.ci.toml index afd4360..9c244cf 100644 --- a/patches/config.ci.toml +++ b/patches/config.ci.toml @@ -7,6 +7,10 @@ tools = ["cargo", "cargo-clippy", "clippy", "rustfmt"] configure-args = [] cargo-native-static = true +[target.riscv32em-athena-zkvm-elf] +cc = "riscv32-unknown-elf-gcc" +cxx = "riscv32-unknown-elf-g++" + [rust] lld = true llvm-tools = true diff --git a/patches/config.toml b/patches/config.toml index 6a085cf..eb771c5 100644 --- a/patches/config.toml +++ b/patches/config.toml @@ -6,6 +6,10 @@ extended = true tools = ["cargo", "cargo-clippy", "clippy", "rustfmt"] configure-args = [] +[target.riscv32em-athena-zkvm-elf] +cc = "riscv32-unknown-elf-gcc" +cxx = "riscv32-unknown-elf-g++" + [rust] lld = true llvm-tools = true diff --git a/patches/llvm.patch b/patches/llvm.patch new file mode 100644 index 0000000..04f24fc --- /dev/null +++ b/patches/llvm.patch @@ -0,0 +1,17 @@ +diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt +index 28ded8766..b51ebe8ea 100644 +--- a/compiler-rt/lib/builtins/CMakeLists.txt ++++ b/compiler-rt/lib/builtins/CMakeLists.txt +@@ -849,9 +849,9 @@ else () + + # For RISCV32, we must force enable int128 for compiling long + # double routines. +- if(COMPILER_RT_ENABLE_SOFTWARE_INT128 OR "${arch}" STREQUAL "riscv32") +- list(APPEND BUILTIN_CFLAGS_${arch} -fforce-enable-int128) +- endif() ++ #if(COMPILER_RT_ENABLE_SOFTWARE_INT128 OR "${arch}" STREQUAL "riscv32") ++ # list(APPEND BUILTIN_CFLAGS_${arch} -fforce-enable-int128) ++ #endif() + + add_compiler_rt_runtime(clang_rt.builtins + STATIC From c3ed1085218298675d282ba57ab0171dfcc543a4 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Mon, 12 Aug 2024 19:47:43 -0700 Subject: [PATCH 05/11] Enable macOS workflows --- .github/workflows/build_and_release.yml | 10 ++++++---- .github/workflows/ci.yml | 8 ++++---- build.sh | 2 -- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml index b8f242f..4da6b5e 100644 --- a/.github/workflows/build_and_release.yml +++ b/.github/workflows/build_and_release.yml @@ -46,7 +46,7 @@ jobs: matrix: os: - ubuntu-latest - # - macos-latest + - macos-latest steps: - name: Free Disk Space (Ubuntu) @@ -63,16 +63,18 @@ jobs: - uses: actions/checkout@v4 - - name: Install ninja + - name: Install ninja and riscv-tools if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt-get -y install ninja-build + sudo apt-get -y install ninja-build binutils-riscv64-unknown-elf gcc-riscv64-unknown-elf - - name: Install ninja + - name: Install ninja and riscv-tools if: contains(matrix.os, 'macos') run: | brew install ninja + brew tap riscv-software-src/riscv + brew install riscv-tools - name: Download source run: ./clone.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f50365d..6ffc14a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,10 +12,10 @@ jobs: include: - os: ubuntu-latest triple: x86_64-unknown-linux-gnu - # - os: macos-14 - # triple: aarch64-apple-darwin - # - os: macos-13 - # triple: x86_64-apple-darwin + - os: macos-14 + triple: aarch64-apple-darwin + - os: macos-13 + triple: x86_64-apple-darwin # - os: buildjet-32vcpu-ubuntu-2004 # triple: x86_64-unknown-linux-gnu # - os: buildjet-32vcpu-ubuntu-2204-arm diff --git a/build.sh b/build.sh index 5f392b3..f91709e 100755 --- a/build.sh +++ b/build.sh @@ -10,8 +10,6 @@ touch /tmp/riscv32em-athena-zkvm-elf.json # Will create component archives (dists) ./rust/build/dist cd rust -#RUSTFLAGS="-C link-arg=-L$HOME/compiler_rt_build_riscv32em-athena-zkvm-elf/lib/baremetal" \ -#LIBRARY_PATH="$HOME/compiler_rt_build_riscv32em-athena-zkvm-elf/lib/baremetal" \ RUST_TARGET_PATH="/tmp" \ CARGO_TARGET_RISCV32EM_ATHENA_ZKVM_ELF_RUSTFLAGS="-Cpasses=loweratomic" \ CFLAGS_riscv32em_athena_zkvm_elf="-ffunction-sections -fdata-sections -fPIC -march=rv32em -mabi=ilp32e" \ From d501daf2ef90f8782e7b568955e46209ea9d2a50 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Mon, 12 Aug 2024 19:57:35 -0700 Subject: [PATCH 06/11] Fix riscv toolchain executable names --- patches/config.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patches/config.toml b/patches/config.toml index eb771c5..1ed3fea 100644 --- a/patches/config.toml +++ b/patches/config.toml @@ -7,8 +7,8 @@ tools = ["cargo", "cargo-clippy", "clippy", "rustfmt"] configure-args = [] [target.riscv32em-athena-zkvm-elf] -cc = "riscv32-unknown-elf-gcc" -cxx = "riscv32-unknown-elf-g++" +cc = "riscv64-unknown-elf-gcc" +cxx = "riscv64-unknown-elf-g++" [rust] lld = true From 05acb50e62f0b81f82e5116aefae13c8291baf5d Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Tue, 13 Aug 2024 14:50:45 -0700 Subject: [PATCH 07/11] Update actions workflow Use self-hosted macOS runner Add timeout Fix issue in package script for macOS --- .github/workflows/build_and_release.yml | 4 +++- package.sh | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml index 4da6b5e..65a0b3b 100644 --- a/.github/workflows/build_and_release.yml +++ b/.github/workflows/build_and_release.yml @@ -40,13 +40,15 @@ jobs: build-and-release: needs: prepare + timeout-minutes: 60 runs-on: ${{ matrix.os }} strategy: matrix: os: + # self-hosted ARM macOS runner + - self-hosted - ubuntu-latest - - macos-latest steps: - name: Free Disk Space (Ubuntu) diff --git a/package.sh b/package.sh index dc40f8d..15302e1 100755 --- a/package.sh +++ b/package.sh @@ -5,5 +5,17 @@ set -euo pipefail source config.sh TOOLCHAIN_DIR=rust/build/$TOOLCHAIN_HOST_TRIPLET/stage2 -cp -n rust/build/$TOOLCHAIN_HOST_TRIPLET/stage2-tools-bin/* $TOOLCHAIN_DIR/bin +TOOLS_BIN_DIR=rust/build/$TOOLCHAIN_HOST_TRIPLET/stage2-tools-bin + +# Function to copy files, ignoring errors if destination exists +copy_ignore_existing() { + cp "$1" "$2" 2>/dev/null || true +} + +# Copy files from stage2-tools-bin to stage2/bin, ignoring if they already exist +for file in "$TOOLS_BIN_DIR"/*; do + copy_ignore_existing "$file" "$TOOLCHAIN_DIR/bin/" +done + +# Create tarball tar --exclude lib/rustlib/src --exclude lib/rustlib/rustc-src -hczvf $ARTIFACT_NAME.tar.gz -C $TOOLCHAIN_DIR . From aa4f8512bfb1604e66eab366a5267b0e8a6416b1 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Tue, 13 Aug 2024 15:08:49 -0700 Subject: [PATCH 08/11] Fail fast if riscv toolchain is missing --- build.sh | 4 ++++ config.sh | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/build.sh b/build.sh index f91709e..2cb9cde 100755 --- a/build.sh +++ b/build.sh @@ -4,6 +4,10 @@ set -euo pipefail source config.sh +# Fail fast if these tools aren't properly installed and in the path +need_cmd riscv64-unknown-elf-gcc +need_cmd riscv64-unknown-elf-g++ + # Rust requires a custom target file to exist for our custom target as part of the bootstrap build, # but it doesn't actually look at the contents. touch /tmp/riscv32em-athena-zkvm-elf.json diff --git a/config.sh b/config.sh index cca11de..bee293c 100755 --- a/config.sh +++ b/config.sh @@ -17,3 +17,26 @@ export ARTIFACT_NAME=athena-rust-toolchain-$TOOLCHAIN_HOST_TRIPLET if [ "$param1" == "-artifact_name" ]; then echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV fi + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (command not found)" + fi +} + +check_cmd() { + command -v "$1" &>/dev/null +} + +say() { + printf "rustc-rv32e-toolchain: %s\n" "$1" +} + +warn() { + say "warning: ${1}" >&2 +} + +err() { + say "$1" >&2 + exit 1 +} From 3c6755f9215283d4ec9dc36357bccc8b522e1861 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Tue, 13 Aug 2024 15:36:24 -0700 Subject: [PATCH 09/11] Remove unneeded package --- .github/workflows/build_and_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml index 65a0b3b..acadc68 100644 --- a/.github/workflows/build_and_release.yml +++ b/.github/workflows/build_and_release.yml @@ -69,7 +69,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt-get -y install ninja-build binutils-riscv64-unknown-elf gcc-riscv64-unknown-elf + sudo apt-get -y install ninja-build gcc-riscv64-unknown-elf - name: Install ninja and riscv-tools if: contains(matrix.os, 'macos') From e6f179e81e72a69b38c681eef2b322198a972058 Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Tue, 13 Aug 2024 16:58:19 -0700 Subject: [PATCH 10/11] Increase workflow timeout --- .github/workflows/build_and_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml index acadc68..267d1b5 100644 --- a/.github/workflows/build_and_release.yml +++ b/.github/workflows/build_and_release.yml @@ -40,7 +40,7 @@ jobs: build-and-release: needs: prepare - timeout-minutes: 60 + timeout-minutes: 120 runs-on: ${{ matrix.os }} strategy: From 5ae09a5d65e8f490e9ec448e1291afb64382282e Mon Sep 17 00:00:00 2001 From: Lane Rettig Date: Tue, 13 Aug 2024 23:34:11 -0400 Subject: [PATCH 11/11] Update config.ci.toml Fix riscv toolchain commands --- patches/config.ci.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patches/config.ci.toml b/patches/config.ci.toml index 9c244cf..7778d89 100644 --- a/patches/config.ci.toml +++ b/patches/config.ci.toml @@ -8,8 +8,8 @@ configure-args = [] cargo-native-static = true [target.riscv32em-athena-zkvm-elf] -cc = "riscv32-unknown-elf-gcc" -cxx = "riscv32-unknown-elf-g++" +cc = "riscv64-unknown-elf-gcc" +cxx = "riscv64-unknown-elf-g++" [rust] lld = true