refactor: remove lfs (#512)

I rebased the PR and removed some more conda-forge repodata.json's that are really large. Instead we download them --------- Co-authored-by: Bas Zalmstra <[email protected]> Co-authored-by: Bas Zalmstra <[email protected]>
conda · Jun 4, 2024 · 182f9a0 · 182f9a0
1 parent bd1c48e
commit 182f9a0
Show file tree

Hide file tree

Showing 82 changed files with 47,003 additions and 1,221 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,3 +0,0 @@
-**/repodata.json filter=lfs diff=lfs merge=lfs -text
-*.conda filter=lfs diff=lfs merge=lfs -text
-*.tar.bz2 filter=lfs diff=lfs merge=lfs -text

diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml
@@ -32,7 +32,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-          lfs: true
       - uses: prefix-dev/[email protected]
         with:
           manifest-path: py-rattler/pixi.toml

diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml
@@ -86,7 +86,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: recursive
-          lfs: ${{ !matrix.skip-tests }}
 
       - name: Install Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1

diff --git a/.gitignore b/.gitignore
@@ -26,5 +26,8 @@ Cargo.lock
 .pixi/
 pixi.lock
 
+# test files that we download
+test-data/channels/conda-forge/
+
 # Visual studio files
 .vs/
diff --git a/crates/rattler/Cargo.toml b/crates/rattler/Cargo.toml
@@ -63,7 +63,7 @@ rstest = { workspace = true }
 tracing-test = { workspace = true }
 insta = { workspace = true, features = ["yaml"] }
 rattler_lock = { path = "../rattler_lock" }
-
+tools = { path="../tools" }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 axum = { workspace = true }
 tower-http = { workspace = true, features = ["fs"] }

diff --git a/crates/rattler/src/install/clobber_registry.rs b/crates/rattler/src/install/clobber_registry.rs
@@ -288,15 +288,21 @@ mod tests {
 
     use crate::install::test_utils::*;
     use crate::{
-        get_repodata_record,
+        get_repodata_record, get_test_data_dir,
         install::{transaction, InstallDriver, InstallOptions, PythonInfo},
         package_cache::PackageCache,
     };
 
     fn test_operations() -> Vec<TransactionOperation<PrefixRecord, RepoDataRecord>> {
-        let repodata_record_1 = get_repodata_record("clobber/clobber-1-0.1.0-h4616a5c_0.tar.bz2");
-        let repodata_record_2 = get_repodata_record("clobber/clobber-2-0.1.0-h4616a5c_0.tar.bz2");
-        let repodata_record_3 = get_repodata_record("clobber/clobber-3-0.1.0-h4616a5c_0.tar.bz2");
+        let repodata_record_1 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-1-0.1.0-h4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_2 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-2-0.1.0-h4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_3 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-3-0.1.0-h4616a5c_0.tar.bz2"),
+        );
 
         vec![
             TransactionOperation::Install(repodata_record_1),
@@ -306,10 +312,12 @@ mod tests {
     }
 
     fn test_python_noarch_operations() -> Vec<TransactionOperation<PrefixRecord, RepoDataRecord>> {
-        let repodata_record_1 =
-            get_repodata_record("clobber/clobber-pynoarch-1-0.1.0-pyh4616a5c_0.tar.bz2");
-        let repodata_record_2 =
-            get_repodata_record("clobber/clobber-pynoarch-2-0.1.0-pyh4616a5c_0.tar.bz2");
+        let repodata_record_1 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-pynoarch-1-0.1.0-pyh4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_2 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-pynoarch-2-0.1.0-pyh4616a5c_0.tar.bz2"),
+        );
 
         vec![
             TransactionOperation::Install(repodata_record_1),
@@ -318,12 +326,15 @@ mod tests {
     }
 
     fn test_operations_nested() -> Vec<TransactionOperation<PrefixRecord, RepoDataRecord>> {
-        let repodata_record_1 =
-            get_repodata_record("clobber/clobber-nested-1-0.1.0-h4616a5c_0.tar.bz2");
-        let repodata_record_2 =
-            get_repodata_record("clobber/clobber-nested-2-0.1.0-h4616a5c_0.tar.bz2");
-        let repodata_record_3 =
-            get_repodata_record("clobber/clobber-nested-3-0.1.0-h4616a5c_0.tar.bz2");
+        let repodata_record_1 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-nested-1-0.1.0-h4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_2 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-nested-2-0.1.0-h4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_3 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-nested-3-0.1.0-h4616a5c_0.tar.bz2"),
+        );
 
         vec![
             TransactionOperation::Install(repodata_record_1),
@@ -333,9 +344,15 @@ mod tests {
     }
 
     fn test_operations_update() -> Vec<RepoDataRecord> {
-        let repodata_record_1 = get_repodata_record("clobber/clobber-1-0.2.0-h4616a5c_0.tar.bz2");
-        let repodata_record_2 = get_repodata_record("clobber/clobber-2-0.2.0-h4616a5c_0.tar.bz2");
-        let repodata_record_3 = get_repodata_record("clobber/clobber-3-0.2.0-h4616a5c_0.tar.bz2");
+        let repodata_record_1 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-1-0.2.0-h4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_2 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-2-0.2.0-h4616a5c_0.tar.bz2"),
+        );
+        let repodata_record_3 = get_repodata_record(
+            get_test_data_dir().join("clobber/clobber-3-0.2.0-h4616a5c_0.tar.bz2"),
+        );
 
         vec![repodata_record_1, repodata_record_2, repodata_record_3]
     }

diff --git a/crates/rattler/src/install/link_script.rs b/crates/rattler/src/install/link_script.rs
@@ -213,7 +213,7 @@ mod tests {
     use rattler_conda_types::{Platform, PrefixRecord, RepoDataRecord};
 
     use crate::{
-        get_repodata_record,
+        get_repodata_record, get_test_data_dir,
         install::{
             test_utils::execute_transaction, transaction, InstallDriver, InstallOptions,
             TransactionOperation,
@@ -222,8 +222,9 @@ mod tests {
     };
 
     fn test_operations() -> Vec<TransactionOperation<PrefixRecord, RepoDataRecord>> {
-        let repodata_record_1 =
-            get_repodata_record("link-scripts/link-scripts-0.1.0-h4616a5c_0.conda");
+        let repodata_record_1 = get_repodata_record(
+            get_test_data_dir().join("link-scripts/link-scripts-0.1.0-h4616a5c_0.conda"),
+        );
 
         vec![TransactionOperation::Install(repodata_record_1)]
     }

diff --git a/crates/rattler/src/install/mod.rs b/crates/rattler/src/install/mod.rs
@@ -849,13 +849,18 @@ mod test {
         let environment_dir = tempfile::TempDir::new().unwrap();
         let package_dir = tempfile::TempDir::new().unwrap();
 
-        // Create package cache
-        rattler_package_streaming::fs::extract(
-            &get_test_data_dir().join("ruff-0.0.171-py310h298983d_0.conda"),
-            package_dir.path(),
+        let package_path = tools::download_and_cache_file_async(
+            "https://conda.anaconda.org/conda-forge/win-64/ruff-0.0.171-py310h298983d_0.conda"
+                .parse()
+                .unwrap(),
+            "25c755b97189ee066576b4ae3999d5e7ff4406d236b984742194e63941838dcd",
         )
+        .await
         .unwrap();
 
+        // Create package cache
+        rattler_package_streaming::fs::extract(&package_path, package_dir.path()).unwrap();
+
         let install_driver = InstallDriver::default();
 
         // Link the package

diff --git a/crates/rattler/src/install/unlink.rs b/crates/rattler/src/install/unlink.rs
@@ -143,23 +143,24 @@ mod tests {
     };
 
     use rattler_conda_types::{Platform, PrefixRecord, RepoDataRecord, Version};
+    use url::Url;
 
     use crate::{
-        get_repodata_record, get_test_data_dir,
+        get_repodata_record,
         install::{
             link_package, unlink_package, InstallDriver, InstallOptions, PythonInfo, Transaction,
         },
     };
 
-    async fn link_ruff(target_prefix: &Path, package: &str) -> PrefixRecord {
+    async fn link_ruff(target_prefix: &Path, package_url: Url, sha256_hash: &str) -> PrefixRecord {
+        let package_path = tools::download_and_cache_file_async(package_url, sha256_hash)
+            .await
+            .unwrap();
+
         let package_dir = tempfile::TempDir::new().unwrap();
 
         // Create package cache
-        rattler_package_streaming::fs::extract(
-            &get_test_data_dir().join(package),
-            package_dir.path(),
-        )
-        .unwrap();
+        rattler_package_streaming::fs::extract(&package_path, package_dir.path()).unwrap();
 
         let py_info =
             PythonInfo::from_version(&Version::from_str("3.10").unwrap(), Platform::Linux64)
@@ -180,7 +181,7 @@ mod tests {
         .await
         .unwrap();
 
-        let repodata_record = get_repodata_record(package);
+        let repodata_record = get_repodata_record(&package_path);
         // Construct a PrefixRecord for the package
 
         PrefixRecord::from_repodata_record(repodata_record, None, None, paths, None, None)
@@ -189,8 +190,14 @@ mod tests {
     #[tokio::test]
     async fn test_unlink_package() {
         let environment_dir = tempfile::TempDir::new().unwrap();
-        let prefix_record =
-            link_ruff(environment_dir.path(), "ruff-0.0.171-py310h298983d_0.conda").await;
+        let prefix_record = link_ruff(
+            environment_dir.path(),
+            "https://conda.anaconda.org/conda-forge/win-64/ruff-0.0.171-py310h298983d_0.conda"
+                .parse()
+                .unwrap(),
+            "25c755b97189ee066576b4ae3999d5e7ff4406d236b984742194e63941838dcd",
+        )
+        .await;
         let conda_meta_path = environment_dir.path().join("conda-meta");
         std::fs::create_dir_all(&conda_meta_path).unwrap();
 
@@ -220,7 +227,8 @@ mod tests {
             .remove_empty_directories(&transaction, &[], environment_dir.path())
             .unwrap();
 
-        // check that the environment is completely empty except for the conda-meta folder
+        // check that the environment is completely empty except for the conda-meta
+        // folder
         let entries = std::fs::read_dir(environment_dir.path())
             .unwrap()
             .collect::<Vec<_>>();
@@ -233,7 +241,10 @@ mod tests {
         let target_prefix = tempfile::TempDir::new().unwrap();
         let prefix_record = link_ruff(
             target_prefix.path(),
-            "pytweening-1.0.4-pyhd8ed1ab_0.tar.bz2",
+            "https://conda.anaconda.org/conda-forge/noarch/pytweening-1.0.4-pyhd8ed1ab_0.tar.bz2"
+                .parse()
+                .unwrap(),
+            "81644bcb60d295f7923770b41daf2d90152ef54b9b094c26513be50fccd62125",
         )
         .await;
 
@@ -278,7 +289,8 @@ mod tests {
             .remove_empty_directories(&transaction, &[], target_prefix.path())
             .unwrap();
 
-        // check that the environment is completely empty except for the conda-meta folder
+        // check that the environment is completely empty except for the conda-meta
+        // folder
         let entries = std::fs::read_dir(target_prefix.path())
             .unwrap()
             .collect::<Vec<_>>();

diff --git a/crates/rattler/src/lib.rs b/crates/rattler/src/lib.rs
@@ -1,13 +1,14 @@
 //! Rattler is an experimental library and executable to work with [Conda](http://conda.io)
-//! environments. Conda is a cross-platform open-source package management system and environment
-//! management system.
+//! environments. Conda is a cross-platform open-source package management
+//! system and environment management system.
 //!
-//! Conda is originally written in Python and has evolved a lot since it was first conceived.
-//! Rattler is an attempt at reimplementing a lot of the machinery supporting Conda but making it
-//! available to a wider range of languages. The goal is to be able to integrate the Conda ecosystem
-//! in a wide variaty of tools that do not rely on Python. Rust has excellent support for
-//! interfacing with many other languages (WASM, Javascript, Python, C, etc) and is therefore a good
-//! candidate for a reimplementation.
+//! Conda is originally written in Python and has evolved a lot since it was
+//! first conceived. Rattler is an attempt at reimplementing a lot of the
+//! machinery supporting Conda but making it available to a wider range of
+//! languages. The goal is to be able to integrate the Conda ecosystem in a wide
+//! variaty of tools that do not rely on Python. Rust has excellent support for
+//! interfacing with many other languages (WASM, Javascript, Python, C, etc) and
+//! is therefore a good candidate for a reimplementation.
 #![deny(missing_docs)]
 
 use std::path::PathBuf;
@@ -18,8 +19,8 @@ pub mod install;
 pub use rattler_cache::package_cache;
 pub use rattler_cache::validation;
 
-/// A helper function that returns a [`Channel`] instance that points to an empty channel on disk
-/// that is bundled with this repository.
+/// A helper function that returns a [`Channel`] instance that points to an
+/// empty channel on disk that is bundled with this repository.
 #[cfg(any(doctest, test))]
 pub fn empty_channel() -> rattler_conda_types::Channel {
     let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -47,20 +48,20 @@ pub fn default_cache_dir() -> anyhow::Result<PathBuf> {
 use rattler_conda_types::RepoDataRecord;
 
 #[cfg(test)]
-pub(crate) fn get_repodata_record(filename: &str) -> RepoDataRecord {
+pub(crate) fn get_repodata_record(package_path: impl AsRef<std::path::Path>) -> RepoDataRecord {
     use std::fs;
 
     use rattler_conda_types::{package::IndexJson, PackageRecord};
     use rattler_digest::{Md5, Sha256};
     use rattler_package_streaming::seek::read_package_file;
 
-    let path = fs::canonicalize(get_test_data_dir().join(filename)).unwrap();
-    let index_json = read_package_file::<IndexJson>(&path).unwrap();
+    let package_path = package_path.as_ref();
+    let index_json = read_package_file::<IndexJson>(&package_path).unwrap();
 
     // find size and hash
-    let size = fs::metadata(&path).unwrap().len();
-    let sha256 = rattler_digest::compute_file_digest::<Sha256>(&path).unwrap();
-    let md5 = rattler_digest::compute_file_digest::<Md5>(&path).unwrap();
+    let size = fs::metadata(&package_path).unwrap().len();
+    let sha256 = rattler_digest::compute_file_digest::<Sha256>(&package_path).unwrap();
+    let md5 = rattler_digest::compute_file_digest::<Md5>(&package_path).unwrap();
 
     RepoDataRecord {
         package_record: PackageRecord::from_index_json(
@@ -70,8 +71,12 @@ pub(crate) fn get_repodata_record(filename: &str) -> RepoDataRecord {
             Some(md5),
         )
         .unwrap(),
-        file_name: filename.to_string(),
-        url: url::Url::from_file_path(&path).unwrap(),
+        file_name: package_path
+            .file_name()
+            .and_then(|f| f.to_str())
+            .unwrap()
+            .to_string(),
+        url: url::Url::from_file_path(&package_path).unwrap(),
         channel: "test".to_string(),
     }
 }
diff --git a/crates/rattler_cache/Cargo.toml b/crates/rattler_cache/Cargo.toml
@@ -18,7 +18,7 @@ itertools.workspace = true
 parking_lot.workspace = true
 rattler_conda_types = { version = "0.25.1", path = "../rattler_conda_types", default-features = false }
 rattler_digest = { version = "0.19.4", path = "../rattler_digest", default-features = false }
-rattler_networking = { version = "0.20.8", path = "../rattler_networking", default-features = false}
+rattler_networking = { version = "0.20.8", path = "../rattler_networking", default-features = false }
 rattler_package_streaming = { version = "0.21.2", path = "../rattler_package_streaming", default-features = false, features = ["reqwest"] }
 reqwest.workspace = true
 tokio.workspace = true
@@ -37,3 +37,4 @@ rstest.workspace = true
 tempfile.workspace = true
 tokio-stream.workspace = true
 tower-http = { workspace = true, features = ["fs"] }
+tools = { path = "../tools" }