From bd218936d22c1074ef59e78e1dd7bf7bc194d163 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 11:38:43 +0100 Subject: [PATCH 1/8] make things compile for wasm again --- crates/rattler_networking/Cargo.toml | 4 +++- .../rattler_networking/src/authentication_middleware.rs | 3 ++- .../src/authentication_storage/backends/file.rs | 8 ++++++++ crates/rattler_package_streaming/Cargo.toml | 9 +++++---- crates/rattler_package_streaming/src/write.rs | 1 + 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index ebc8d3b90..e7c32eb87 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,15 +11,17 @@ license.workspace = true readme.workspace = true [features] +default = ["fslock"] native-tls = ['reqwest/native-tls'] rustls-tls = ['reqwest/rustls-tls'] +wasm = [] [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } dirs = { workspace = true } -fslock = { workspace = true } +fslock = { workspace = true, optional = true } itertools = { workspace = true } keyring = { workspace = true } lazy_static = { workspace = true } diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index a60514c19..d56d675c7 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -16,7 +16,8 @@ pub struct AuthenticationMiddleware { auth_storage: AuthenticationStorage, } -#[async_trait] +#[cfg_attr(not(feature = "wasm"), async_trait)] +#[cfg_attr(feature = "wasm", async_trait(?Send))] impl Middleware for AuthenticationMiddleware { async fn handle( &self, diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index ce640f3f1..0318b6658 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,5 +1,6 @@ //! file storage for passwords. use anyhow::Result; +#[cfg(not(feature = "wasm"))] use fslock::LockFile; use once_cell::sync::Lazy; use std::collections::{BTreeMap, HashSet}; @@ -38,6 +39,7 @@ impl FileStorage { Self { path } } + #[cfg(not(feature = "wasm"))] /// Lock the file storage file for reading and writing. This will block until the lock is /// acquired. fn lock(&self) -> Result { @@ -60,6 +62,12 @@ impl FileStorage { Ok(lock) } + #[cfg(feature = "wasm")] + /// Fake lock + fn lock(&self) -> Result<(), FileStorageError> { + Ok(()) + } + /// Read the JSON file and deserialize it into a `BTreeMap`, or return an empty `BTreeMap` if the /// file does not exist fn read_json(&self) -> Result, FileStorageError> { diff --git a/crates/rattler_package_streaming/Cargo.toml b/crates/rattler_package_streaming/Cargo.toml index 6573af264..9038df179 100644 --- a/crates/rattler_package_streaming/Cargo.toml +++ b/crates/rattler_package_streaming/Cargo.toml @@ -25,17 +25,18 @@ serde_json = { workspace = true } tar = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } +tokio = { workspace = true, default-features = false } tokio-util = { workspace = true, features = ["io-util"] } url = { workspace = true } zip = { workspace = true, features = ["deflate", "time"] } -zstd = { workspace = true, features = ["zstdmt"] } +zstd = { workspace = true } [features] -default = ["native-tls"] +default = ["native-tls", "tokio/fs", "zstdmt"] +zstdmt = ["zstd/zstdmt"] native-tls = ["rattler_networking/native-tls"] rustls-tls = ["rattler_networking/rustls-tls"] -wasm = ["zstd/wasm"] +wasm = ["zstd/wasm", "rattler_networking/wasm"] reqwest = ["dep:reqwest-middleware", "dep:reqwest"] [dev-dependencies] diff --git a/crates/rattler_package_streaming/src/write.rs b/crates/rattler_package_streaming/src/write.rs index 6e5f62b01..9ed5ed3c3 100644 --- a/crates/rattler_package_streaming/src/write.rs +++ b/crates/rattler_package_streaming/src/write.rs @@ -239,6 +239,7 @@ fn write_zst_archive( let tar_file = File::open(&tar_path)?; let compression_level = compression_level.to_zstd_level()?; let mut zst_encoder = zstd::Encoder::new(writer, compression_level)?; + #[cfg(feature = "zstdmt")] zst_encoder.multithread(num_threads.unwrap_or_else(|| num_cpus::get() as u32))?; progress_bar_wrapper.reset_position(); From 5cd786834dfc28ad26cb47a30ad54582964879c6 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 29 Feb 2024 15:52:27 +0100 Subject: [PATCH 2/8] fix: make it compile --- .../src/authentication_storage/backends/file.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 0318b6658..99077b1b8 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,6 +1,6 @@ //! file storage for passwords. use anyhow::Result; -#[cfg(not(feature = "wasm"))] +#[cfg(feature="fslock")] use fslock::LockFile; use once_cell::sync::Lazy; use std::collections::{BTreeMap, HashSet}; @@ -39,7 +39,7 @@ impl FileStorage { Self { path } } - #[cfg(not(feature = "wasm"))] + #[cfg(feature="fslock")] /// Lock the file storage file for reading and writing. This will block until the lock is /// acquired. fn lock(&self) -> Result { @@ -62,7 +62,7 @@ impl FileStorage { Ok(lock) } - #[cfg(feature = "wasm")] + #[cfg(not(feature="fslock"))] /// Fake lock fn lock(&self) -> Result<(), FileStorageError> { Ok(()) From dcbd92233128dffee4d1ea2951ce10497145fa0c Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 16:07:37 +0100 Subject: [PATCH 3/8] fmt --- .../src/authentication_storage/backends/file.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 99077b1b8..9f8a299bf 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,6 +1,6 @@ //! file storage for passwords. use anyhow::Result; -#[cfg(feature="fslock")] +#[cfg(feature = "fslock")] use fslock::LockFile; use once_cell::sync::Lazy; use std::collections::{BTreeMap, HashSet}; @@ -39,7 +39,7 @@ impl FileStorage { Self { path } } - #[cfg(feature="fslock")] + #[cfg(feature = "fslock")] /// Lock the file storage file for reading and writing. This will block until the lock is /// acquired. fn lock(&self) -> Result { @@ -62,7 +62,7 @@ impl FileStorage { Ok(lock) } - #[cfg(not(feature="fslock"))] + #[cfg(not(feature = "fslock"))] /// Fake lock fn lock(&self) -> Result<(), FileStorageError> { Ok(()) From 52d569954a2d8662a52374cb4288448abeae5903 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 16:19:47 +0100 Subject: [PATCH 4/8] try to compile wasm in ci --- .github/workflows/rust-compile.yml | 11 +++++++++++ crates/rattler/Cargo.toml | 2 +- crates/rattler_conda_types/Cargo.toml | 2 +- crates/rattler_index/Cargo.toml | 2 +- crates/rattler_package_streaming/src/write.rs | 3 +++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 8cf5468cf..a74483e49 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -81,6 +81,7 @@ jobs: - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } + - { name: "Webassembly", target: wasm32-unknown-unknown, os: ubuntu-latest, skip-tests: true } steps: - name: Checkout source code uses: actions/checkout@v4 @@ -123,6 +124,7 @@ jobs: echo "CARGO_BUILD_OPTIONS=${CARGO_BUILD_OPTIONS} --no-default-features --features rustls-tls" >> $GITHUB_OUTPUT - name: Build + if: matrix.target != 'wasm32-unknown-unknown' run: > cargo build --all-targets @@ -130,6 +132,15 @@ jobs: --target ${{ matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} + - name: Build + if: matrix.target == 'wasm32-unknown-unknown' + run: | + cd crates/rattler_package_streaming + cargo build \ + --all-targets \ + --features wasm \ + --target ${{ matrix.target }} + - name: Disable testing the tools crate if cross compiling id: test-options if: ${{ !matrix.skip-tests }} diff --git a/crates/rattler/Cargo.toml b/crates/rattler/Cargo.toml index 5abd5597b..2bf8a8a22 100644 --- a/crates/rattler/Cargo.toml +++ b/crates/rattler/Cargo.toml @@ -39,7 +39,7 @@ pin-project-lite = { workspace = true } rattler_conda_types = { path="../rattler_conda_types", version = "0.19.0", default-features = false } rattler_digest = { path="../rattler_digest", version = "0.19.0", default-features = false } rattler_networking = { path="../rattler_networking", version = "0.19.0", default-features = false } -rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false, features = ["reqwest"] } +rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false, features = ["reqwest", "zstdmt"] } reflink-copy = { workspace = true } regex = { workspace = true } reqwest = { workspace = true, features = ["stream", "json", "gzip"] } diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 60b869832..1f23964ac 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -37,7 +37,7 @@ url = { workspace = true, features = ["serde"] } [dev-dependencies] rand = { workspace = true } insta = { workspace = true, features = ["yaml", "redactions", "toml"] } -rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls"] } +rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls", "zstdmt"] } tempfile = { workspace = true } rstest = { workspace = true } assert_matches = { workspace = true } diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index 08fb14d49..7481a274d 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -14,7 +14,7 @@ readme.workspace = true fs-err = { workspace = true } rattler_conda_types = { path="../rattler_conda_types", version = "0.19.0", default-features = false } rattler_digest = { path="../rattler_digest", version = "0.19.0", default-features = false } -rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false } +rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false, features = ["zstdmt"] } serde_json = { workspace = true } tracing = { workspace = true } walkdir = { workspace = true } diff --git a/crates/rattler_package_streaming/src/write.rs b/crates/rattler_package_streaming/src/write.rs index 9ed5ed3c3..9fcca00ec 100644 --- a/crates/rattler_package_streaming/src/write.rs +++ b/crates/rattler_package_streaming/src/write.rs @@ -241,6 +241,9 @@ fn write_zst_archive( let mut zst_encoder = zstd::Encoder::new(writer, compression_level)?; #[cfg(feature = "zstdmt")] zst_encoder.multithread(num_threads.unwrap_or_else(|| num_cpus::get() as u32))?; + // mark as "used" to avoid "unused" warning + #[cfg(not(feature = "zstdmt"))] + let _ = num_threads; progress_bar_wrapper.reset_position(); if let Ok(tar_total_size) = tar_file.metadata().map(|v| v.len()) { From 2f38dc999ae59af631677de68dc9dd6934e8daf0 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 16:25:44 +0100 Subject: [PATCH 5/8] .. --- .github/workflows/rust-compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index a74483e49..6d772e21e 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -97,7 +97,7 @@ jobs: cache: false - uses: taiki-e/setup-cross-toolchain-action@v1 - if: matrix.target != 'x86_64-unknown-linux-musl' + if: !(matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'wasm32-unknown-unknown') with: target: ${{ matrix.target }} From b6fe9d5d00c9431f13fb6128e2fc6b55721b76da Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 16:25:58 +0100 Subject: [PATCH 6/8] debug --- .github/workflows/rust-compile.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 6d772e21e..05f212906 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -61,26 +61,26 @@ jobs: fail-fast: false matrix: include: - - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-latest } - - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } - - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-latest } +# - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } +# - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } -# - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } +# - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } +# - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } - - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } + # - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } + # - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } - { name: "Webassembly", target: wasm32-unknown-unknown, os: ubuntu-latest, skip-tests: true } steps: - name: Checkout source code From a04e90d391f0a7219cefbc7639d88a681b177396 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 16:46:18 +0100 Subject: [PATCH 7/8] fix condition --- .github/workflows/rust-compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 05f212906..fa088fbad 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -97,7 +97,7 @@ jobs: cache: false - uses: taiki-e/setup-cross-toolchain-action@v1 - if: !(matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'wasm32-unknown-unknown') + if: ${{ !(matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'wasm32-unknown-unknown') }} with: target: ${{ matrix.target }} From 7cf9e5ea5a504c0b00f3117744c6898873843225 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 29 Feb 2024 16:50:10 +0100 Subject: [PATCH 8/8] .. --- .github/workflows/rust-compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index fa088fbad..8aab4dff9 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -137,7 +137,7 @@ jobs: run: | cd crates/rattler_package_streaming cargo build \ - --all-targets \ + --no-default-features \ --features wasm \ --target ${{ matrix.target }}