feat: always use urls for channels (#886)

* feat: always use urls for channels

* fix: clippy

* fix: don't touch pixi version

* fix: end-to-end test

* fix: end-to-end test

* fix: end-to-end test

.github/workflows/end-to-end.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [main]
+    branches: [ main ]
     paths:
       - "src/**"
       - "Cargo.*"
@@ -28,7 +28,7 @@ jobs:
   run_tests:
     strategy:
       matrix:
-        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        os: [ "ubuntu-latest", "windows-latest", "macos-latest" ]
       fail-fast: false
 
     name: Run tests
@@ -41,8 +41,6 @@ jobs:
         with:
           toolchain: "1.75.0"
       - uses: prefix-dev/[email protected]
-        with:
-          pixi-version: v0.13.0
       - if: runner.os == 'macOS'
         run: pixi global install patchelf
       # build in release mode so that it's reasonably fast to run the tests

pixi.toml

@@ -31,6 +31,7 @@ perl = "5.32.1.*"
 pytest = "7.4.2.*"
 pyyaml = ">=6.0.1,<6.1"
 conda-package-handling = "2.2.0.*"
+requests = ">=2.32.2,<2.33"
 
 [feature.docs.dependencies]
 mkdocs = "1.5.3.*"
@@ -42,4 +43,4 @@ mike = "2.0.0.*"
 [environments]
 # Using same solve group to keep the environment consistent in versions used and improving cache hits
 default = {solve-group = "default"}
-docs = {features = ["docs"], no-default-feature = true, solve-group = "default"}
+docs = {features = ["docs"], no-default-feature = true, solve-group = "default"}

rust-tests/src/lib.rs

@@ -367,7 +367,7 @@ mod tests {
                     }
                 }
             } else if actual.ne(&cmp) {
-                panic!("Expected {f} to be {cmp:?} but was {actual:?}");
+                panic!("Mismatch in {f}:\nExpected:\n  {cmp:?}{f}\nActual:\n  {actual:?}");
             }
         }
     }

src/build.rs

@@ -1,5 +1,5 @@
 //! The build module contains the code for running the build process for a given [`Output`]
-use rattler_conda_types::{MatchSpec, ParseStrictness};
+use rattler_conda_types::{Channel, MatchSpec, ParseStrictness};
 use std::path::PathBuf;
 use std::vec;
 
@@ -42,12 +42,10 @@ pub async fn skip_existing(
         .collect::<Result<Vec<_>, _>>()?;
 
     let channels = if only_local {
-        vec![first_output
-            .build_configuration
-            .directories
-            .output_dir
-            .to_string_lossy()
-            .to_string()]
+        vec![
+            Channel::from_directory(&first_output.build_configuration.directories.output_dir)
+                .base_url,
+        ]
     } else {
         all_channels
     };

src/lib.rs

@@ -36,9 +36,9 @@ use build::skip_existing;
 use dunce::canonicalize;
 use fs_err as fs;
 use metadata::Output;
-use miette::IntoDiagnostic;
+use miette::{IntoDiagnostic, WrapErr};
 use petgraph::{algo::toposort, graph::DiGraph, visit::DfsPostOrder};
-use rattler_conda_types::{package::ArchiveType, Platform};
+use rattler_conda_types::{package::ArchiveType, Channel, ChannelConfig, Platform};
 use std::{
     collections::{BTreeMap, HashMap},
     env::current_dir,
@@ -47,8 +47,6 @@ use std::{
 };
 use tool_configuration::Configuration;
 
-use crate::tool_configuration::SkipExisting;
-
 use {
     build::run_build,
     console_utils::LoggingOutputHandler,
@@ -129,6 +127,7 @@ pub fn get_tool_config(
         use_bz2: args.common.use_bz2,
         render_only: args.render_only,
         skip_existing: args.skip_existing,
+        ..Configuration::default()
     })
 }
 
@@ -252,10 +251,15 @@ pub async fn get_build_output(
 
         let name = recipe.package().name().clone();
         // Add the channels from the args and by default always conda-forge
+
         let channels = args
             .channel
             .clone()
-            .unwrap_or_else(|| vec!["conda-forge".to_string()]);
+            .unwrap_or_else(|| vec!["conda-forge".to_string()])
+            .into_iter()
+            .map(|c| Channel::from_str(c, &tool_config.channel_config).map(|c| c.base_url))
+            .collect::<Result<Vec<_>, _>>()
+            .into_diagnostic()?;
 
         let timestamp = chrono::Utc::now();
 
@@ -349,15 +353,27 @@ pub async fn run_test_from_args(
     let client = tool_configuration::reqwest_client_from_auth_storage(args.common.auth_file)
         .into_diagnostic()?;
 
+    let channel_config = ChannelConfig::default_with_root_dir(
+        std::env::current_dir()
+            .into_diagnostic()
+            .context("failed to determine the current directory")?,
+    );
+
+    let channels = args
+        .channel
+        .unwrap_or_else(|| vec!["conda-forge".to_string()])
+        .into_iter()
+        .map(|name| Channel::from_str(name, &channel_config).map(|c| c.base_url))
+        .collect::<Result<Vec<_>, _>>()
+        .into_diagnostic()?;
+
     let tempdir = tempfile::tempdir().into_diagnostic()?;
 
     let test_options = TestConfiguration {
         test_prefix: tempdir.path().to_path_buf(),
         target_platform: None,
         keep_test_prefix: false,
-        channels: args
-            .channel
-            .unwrap_or_else(|| vec!["conda-forge".to_string()]),
+        channels,
         tool_configuration: Configuration {
             client,
             fancy_log_handler,
@@ -426,8 +442,7 @@ pub async fn rebuild_from_args(
         no_test: args.no_test,
         use_zstd: args.common.use_zstd,
         use_bz2: args.common.use_bz2,
-        render_only: false,
-        skip_existing: SkipExisting::None,
+        ..Configuration::default()
     };
 
     output

src/metadata.rs

@@ -3,6 +3,7 @@ use std::{
     collections::BTreeMap,
     fmt::{self, Display, Formatter},
     io::Write,
+    iter,
     path::{Path, PathBuf},
     str::FromStr,
     sync::{Arc, Mutex},
@@ -14,11 +15,12 @@ use fs_err as fs;
 use indicatif::HumanBytes;
 use rattler_conda_types::{
     package::{ArchiveType, PathType, PathsEntry, PathsJson},
-    PackageName, Platform, RepoDataRecord,
+    Channel, PackageName, Platform, RepoDataRecord,
 };
 use rattler_index::index;
 use rattler_package_streaming::write::CompressionLevel;
 use serde::{Deserialize, Serialize};
+use url::Url;
 
 use crate::{
     console_utils::github_integration_enabled,
@@ -237,7 +239,7 @@ pub struct BuildConfiguration {
     /// The directories for the build (work, source, build, host, ...)
     pub directories: Directories,
     /// The channels to use when resolving environments
-    pub channels: Vec<String>,
+    pub channels: Vec<Url>,
     /// The timestamp to use for the build
     pub timestamp: chrono::DateTime<chrono::Utc>,
     /// All subpackages coming from this output or other outputs from the same recipe
@@ -327,14 +329,14 @@ impl Output {
     }
 
     /// The channels to use when resolving dependencies
-    pub fn reindex_channels(&self) -> Result<Vec<String>, std::io::Error> {
+    pub fn reindex_channels(&self) -> Result<Vec<Url>, std::io::Error> {
         let output_dir = &self.build_configuration.directories.output_dir;
 
         index(output_dir, Some(&self.build_configuration.target_platform))?;
 
-        let mut channels = vec![output_dir.to_string_lossy().to_string()];
-        channels.extend(self.build_configuration.channels.clone());
-        Ok(channels)
+        Ok(iter::once(Channel::from_directory(output_dir).base_url)
+            .chain(self.build_configuration.channels.iter().cloned())
+            .collect())
     }
 
     /// retrieve an identifier for this output ({name}-{version}-{build_string})

src/opt.rs

@@ -205,7 +205,7 @@ pub struct BuildOpts {
     #[arg(long, default_value_t = Platform::current())]
     pub target_platform: Platform,
 
-    /// Add the channels needed for the recipe using this option. For more then one channel use it multiple times.
+    /// Add the channels needed for the recipe using this option. For more than one channel use it multiple times.
     /// The default channel is `conda-forge`.
     #[arg(short = 'c', long)]
     pub channel: Option<Vec<String>>,

src/package_test/run_test.rs

@@ -9,7 +9,7 @@
 
 use fs_err as fs;
 use rattler_conda_types::package::IndexJson;
-use rattler_conda_types::ParseStrictness;
+use rattler_conda_types::{Channel, ParseStrictness};
 use std::fmt::Write as fmt_write;
 use std::{
     path::{Path, PathBuf},
@@ -21,6 +21,7 @@ use rattler::package_cache::CacheKey;
 use rattler_conda_types::{package::ArchiveIdentifier, MatchSpec, Platform};
 use rattler_index::index;
 use rattler_shell::activation::ActivationError;
+use url::Url;
 
 use crate::env_vars;
 use crate::recipe::parser::{Script, ScriptContent};
@@ -177,7 +178,7 @@ pub struct TestConfiguration {
     pub keep_test_prefix: bool,
     /// The channels to use for the test – do not forget to add the local build outputs channel
     /// if desired
-    pub channels: Vec<String>,
+    pub channels: Vec<Url>,
     /// The tool configuration
     pub tool_configuration: tool_configuration::Configuration,
 }
@@ -261,7 +262,7 @@ pub async fn run_test(package_file: &Path, config: &TestConfiguration) -> Result
     };
 
     let mut channels = config.channels.clone();
-    channels.insert(0, tmp_repo.path().to_string_lossy().to_string());
+    channels.insert(0, Channel::from_directory(tmp_repo.path()).base_url);
 
     let config = TestConfiguration {
         target_platform: Some(target_platform),

src/packaging/metadata.rs

@@ -237,7 +237,12 @@ impl Output {
                 .unwrap_or_default(),
             // TODO ?
             source_url: None,
-            channels: self.build_configuration.channels.clone(),
+            channels: self
+                .build_configuration
+                .channels
+                .iter()
+                .map(|c| c.to_string())
+                .collect(),
         };
 
         about_json

src/render/resolved_dependencies.rs

@@ -26,6 +26,7 @@ use crate::recipe::parser::Dependency;
 use crate::render::pin::PinArgs;
 use crate::render::solver::install_packages;
 use serde_with::{serde_as, DisplayFromStr};
+use url::Url;
 
 /// A enum to keep track of where a given Dependency comes from
 #[serde_as]
@@ -654,7 +655,7 @@ pub async fn install_environments(
 #[allow(clippy::for_kv_map)]
 async fn resolve_dependencies(
     output: &Output,
-    channels: &[String],
+    channels: &[Url],
     tool_configuration: &tool_configuration::Configuration,
 ) -> Result<FinalizedDependencies, ResolveError> {
     let merge_build_host = output.recipe.build().merge_build_and_host_envs();

src/render/solver.rs

@@ -8,8 +8,7 @@ use rattler::{
     package_cache::PackageCache,
 };
 use rattler_conda_types::{
-    Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, Platform, PrefixRecord,
-    RepoDataRecord,
+    Channel, GenericVirtualPackage, MatchSpec, Platform, PrefixRecord, RepoDataRecord,
 };
 use rattler_repodata_gateway::sparse::SparseRepoData;
 use rattler_repodata_gateway::{
@@ -70,7 +69,7 @@ pub async fn create_environment(
     specs: &[MatchSpec],
     target_platform: &Platform,
     target_prefix: &Path,
-    channels: &[String],
+    channels: &[Url],
     tool_configuration: &tool_configuration::Configuration,
 ) -> anyhow::Result<Vec<RepoDataRecord>> {
     // Parse the specs from the command line. We do this explicitly instead of allow clap to deal
@@ -80,7 +79,10 @@ pub async fn create_environment(
     tracing::info!("  Platform: {}", target_platform);
     tracing::info!("  Channels: ");
     for channel in channels {
-        tracing::info!("   - {}", channel);
+        tracing::info!(
+            "   - {}",
+            tool_configuration.channel_config.canonical_name(channel)
+        );
     }
     tracing::info!("  Specs:");
     for spec in specs {
@@ -150,21 +152,15 @@ pub async fn create_environment(
 
 /// Load repodata for given matchspecs and channels.
 pub async fn load_repodatas(
-    channels: &[String],
+    channels: &[Url],
     target_platform: &Platform,
     tool_configuration: &tool_configuration::Configuration,
     specs: &[MatchSpec],
 ) -> Result<(PathBuf, Vec<Vec<RepoDataRecord>>), anyhow::Error> {
-    let channel_config = ChannelConfig::default_with_root_dir(std::env::current_dir()?);
     let cache_dir = rattler::default_cache_dir()?;
     std::fs::create_dir_all(&cache_dir)
         .map_err(|e| anyhow::anyhow!("could not create cache directory: {}", e))?;
 
-    let channels = channels
-        .iter()
-        .map(|channel_str| Channel::from_str(channel_str, &channel_config))
-        .collect::<Result<Vec<_>, _>>()?;
-
     let platforms = [Platform::NoArch, *target_platform];
     let channel_urls = channels
         .iter()
@@ -180,12 +176,12 @@ pub async fn load_repodatas(
     let channel_and_platform_len = channel_urls.len();
     let repodata_download_client = tool_configuration.client.clone();
     let sparse_repo_datas = futures::stream::iter(channel_urls)
-        .map(move |(channel, platform)| {
+        .map(move |(url, platform)| {
             let repodata_cache = repodata_cache_path.clone();
             let download_client = repodata_download_client.clone();
             async move {
                 fetch_repo_data_records_with_progress(
-                    channel,
+                    Channel::from_url(url),
                     platform,
                     &repodata_cache,
                     download_client.clone(),

src/snapshots/rattler_build__metadata__test__curl_recipe.yaml.snap

@@ -1,6 +1,6 @@
 ---
 source: src/metadata.rs
-assertion_line: 730
+assertion_line: 732
 expression: output
 ---
 recipe:
@@ -44,7 +44,7 @@ build_configuration:
     work_dir: /Users/wolfv/Programs/rattler-build/output/bld/rattler-build_curl_1700573293/work
     build_dir: /Users/wolfv/Programs/rattler-build/output/bld/rattler-build_curl_1700573293
   channels:
-    - conda-forge
+    - "https://conda.anaconda.org/conda-forge"
   timestamp: "2023-11-21T13:28:13.086796Z"
   subpackages:
     curl:

src/snapshots/rattler_build__metadata__test__read_recipe_with_sources.snap

@@ -1,6 +1,6 @@
 ---
 source: src/metadata.rs
-assertion_line: 744
+assertion_line: 742
 expression: git_source_output
 ---
 recipe:
@@ -40,7 +40,7 @@ build_configuration:
     work_dir: /Users/wolfv/Programs/rattler-build/output/bld/rattler-build_git_source_1704379826/work
     build_dir: /Users/wolfv/Programs/rattler-build/output/bld/rattler-build_git_source_1704379826
   channels:
-    - conda-forge
+    - "https://conda.anaconda.org/conda-forge"
   timestamp: "2024-01-04T14:50:26.411202Z"
   subpackages:
     git_source: