feat: Add support for optional dependencies (#1019)

Co-authored-by: Bas Zalmstra <[email protected]>

.github/workflows/rust-compile.yml

@@ -146,7 +146,7 @@ jobs:
         run: >
           cargo nextest run
           --workspace 
-          --features ${{ env.DEFAULT_FEATURES }} 
+          --features ${{ env.DEFAULT_FEATURES }},experimental_extras
           --target ${{ matrix.target }}
           ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} 
           ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} 

crates/rattler-bin/src/commands/create.rs

@@ -1,5 +1,6 @@
 use std::{
     borrow::Cow,
+    collections::HashMap,
     env,
     future::IntoFuture,
     path::PathBuf,
@@ -18,8 +19,8 @@ use rattler::{
     package_cache::PackageCache,
 };
 use rattler_conda_types::{
-    Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform,
-    PrefixRecord, RepoDataRecord, Version,
+    Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, ParseStrictness,
+    Platform, PrefixRecord, RepoDataRecord, Version,
 };
 use rattler_networking::AuthenticationMiddleware;
 use rattler_repodata_gateway::{Gateway, RepoData, SourceConfig};
@@ -258,12 +259,14 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
     // Next, use a solver to solve this specific problem. This provides us with all
     // the operations we need to apply to our environment to bring it up to
     // date.
-    let required_packages =
+    let solver_result =
         wrap_in_progress("solving", move || match opt.solver.unwrap_or_default() {
             Solver::Resolvo => resolvo::Solver.solve(solver_task),
             Solver::LibSolv => libsolv_c::Solver.solve(solver_task),
         })?;
 
+    let required_packages: Vec<RepoDataRecord> = solver_result.records;
+
     if opt.dry_run {
         // Construct a transaction to
         let transaction = Transaction::from_current_and_desired(
@@ -275,7 +278,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         if transaction.operations.is_empty() {
             println!("No operations necessary");
         } else {
-            print_transaction(&transaction);
+            print_transaction(&transaction, solver_result.features);
         }
 
         return Ok(());
@@ -306,28 +309,42 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
             console::style(console::Emoji("✔", "")).green(),
             install_start.elapsed()
         );
-        print_transaction(&result.transaction);
+        print_transaction(&result.transaction, solver_result.features);
     }
 
     Ok(())
 }
 
 /// Prints the operations of the transaction to the console.
-fn print_transaction(transaction: &Transaction<PrefixRecord, RepoDataRecord>) {
+fn print_transaction(
+    transaction: &Transaction<PrefixRecord, RepoDataRecord>,
+    features: HashMap<PackageName, Vec<String>>,
+) {
     let format_record = |r: &RepoDataRecord| {
         let direct_url_print = if let Some(channel) = &r.channel {
             channel.clone()
         } else {
             String::new()
         };
 
-        format!(
-            "{} {} {} {}",
-            r.package_record.name.as_normalized(),
-            r.package_record.version,
-            r.package_record.build,
-            direct_url_print,
-        )
+        if let Some(features) = features.get(&r.package_record.name) {
+            format!(
+                "{}[{}] {} {} {}",
+                r.package_record.name.as_normalized(),
+                features.join(", "),
+                r.package_record.version,
+                r.package_record.build,
+                direct_url_print,
+            )
+        } else {
+            format!(
+                "{} {} {} {}",
+                r.package_record.name.as_normalized(),
+                r.package_record.version,
+                r.package_record.build,
+                direct_url_print,
+            )
+        }
     };
 
     for operation in &transaction.operations {

crates/rattler_conda_types/Cargo.toml

@@ -12,6 +12,7 @@ readme.workspace = true
 
 [features]
 default = ["rayon"]
+experimental_extras = []
 
 [dependencies]
 chrono = { workspace = true }

crates/rattler_conda_types/src/lib.rs

@@ -27,7 +27,7 @@ pub mod prefix_record;
 #[cfg(test)]
 use std::path::{Path, PathBuf};
 
-pub use build_spec::{BuildNumber, BuildNumberSpec, ParseBuildNumberSpecError};
+pub use build_spec::{BuildNumber, BuildNumberSpec, OrdOperator, ParseBuildNumberSpecError};
 pub use channel::{Channel, ChannelConfig, ChannelUrl, NamedChannelOrUrl, ParseChannelError};
 pub use channel_data::{ChannelData, ChannelDataPackage};
 pub use environment_yaml::{EnvironmentYaml, MatchSpecOrSubSection};
@@ -53,7 +53,7 @@ pub use repo_data::{
     ChannelInfo, ConvertSubdirError, PackageRecord, RecordFromPath, RepoData,
     ValidatePackageRecordsError,
 };
-pub use repo_data_record::RepoDataRecord;
+pub use repo_data_record::{RepoDataRecord, SolverResult};
 pub use run_export::RunExportKind;
 pub use version::{
     Component, ParseVersionError, ParseVersionErrorKind, StrictVersion, Version, VersionBumpError,

crates/rattler_conda_types/src/match_spec/mod.rs

@@ -2,6 +2,7 @@ use crate::{
     build_spec::BuildNumberSpec, GenericVirtualPackage, PackageName, PackageRecord, RepoDataRecord,
     VersionSpec,
 };
+use itertools::Itertools;
 use rattler_digest::{serde::SerializableHash, Md5Hash, Sha256Hash};
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_with::{serde_as, skip_serializing_none};
@@ -135,6 +136,8 @@ pub struct MatchSpec {
     pub build_number: Option<BuildNumberSpec>,
     /// Match the specific filename of the package
     pub file_name: Option<String>,
+    /// The selected optional features of the package
+    pub extras: Option<Vec<String>>,
     /// The channel of the package
     pub channel: Option<Arc<Channel>>,
     /// The subdir of the channel
@@ -183,6 +186,10 @@ impl Display for MatchSpec {
 
         let mut keys = Vec::new();
 
+        if let Some(extras) = &self.extras {
+            keys.push(format!("extras=[{}]", extras.iter().format(", ")));
+        }
+
         if let Some(md5) = &self.md5 {
             keys.push(format!("md5=\"{md5:x}\""));
         }
@@ -221,6 +228,7 @@ impl MatchSpec {
                 build: self.build,
                 build_number: self.build_number,
                 file_name: self.file_name,
+                extras: self.extras,
                 channel: self.channel,
                 subdir: self.subdir,
                 namespace: self.namespace,
@@ -265,6 +273,8 @@ pub struct NamelessMatchSpec {
     pub build_number: Option<BuildNumberSpec>,
     /// Match the specific filename of the package
     pub file_name: Option<String>,
+    /// Optional extra dependencies to select for the package
+    pub extras: Option<Vec<String>>,
     /// The channel of the package
     #[serde(deserialize_with = "deserialize_channel", default)]
     pub channel: Option<Arc<Channel>>,
@@ -318,6 +328,7 @@ impl From<MatchSpec> for NamelessMatchSpec {
             build: spec.build,
             build_number: spec.build_number,
             file_name: spec.file_name,
+            extras: spec.extras,
             channel: spec.channel,
             subdir: spec.subdir,
             namespace: spec.namespace,
@@ -337,6 +348,7 @@ impl MatchSpec {
             build: spec.build,
             build_number: spec.build_number,
             file_name: spec.file_name,
+            extras: spec.extras,
             channel: spec.channel,
             subdir: spec.subdir,
             namespace: spec.namespace,

crates/rattler_conda_types/src/match_spec/parse.rs

@@ -10,6 +10,7 @@ use nom::{
     sequence::{delimited, preceded, separated_pair, terminated},
     Finish, IResult,
 };
+
 use rattler_digest::{parse_digest_from_hex, Md5, Sha256};
 use smallvec::SmallVec;
 use thiserror::Error;
@@ -178,6 +179,7 @@ fn parse_bracket_list(input: &str) -> Result<BracketVec<'_>, ParseMatchSpecError
             alt((
                 delimited(char('"'), take_until("\""), char('"')),
                 delimited(char('\''), take_until("'"), char('\'')),
+                delimited(char('['), take_until("]"), char(']')),
                 take_till1(|c| c == ',' || c == ']' || c == '\'' || c == '"'),
             )),
         ))(input)
@@ -207,7 +209,9 @@ fn parse_bracket_list(input: &str) -> Result<BracketVec<'_>, ParseMatchSpecError
 /// Strips the brackets part of the matchspec returning the rest of the
 /// matchspec and  the contents of the brackets as a `Vec<&str>`.
 fn strip_brackets(input: &str) -> Result<(Cow<'_, str>, BracketVec<'_>), ParseMatchSpecError> {
-    if let Some(matches) = lazy_regex::regex!(r#".*(?:(\[.*\]))$"#).captures(input) {
+    if let Some(matches) =
+        lazy_regex::regex!(r#".*(\[(?:[^\[\]]|\[(?:[^\[\]]|\[.*\])*\])*\])$"#).captures(input)
+    {
         let bracket_str = matches.get(1).unwrap().as_str();
         let bracket_contents = parse_bracket_list(bracket_str)?;
 
@@ -223,6 +227,32 @@ fn strip_brackets(input: &str) -> Result<(Cow<'_, str>, BracketVec<'_>), ParseMa
     }
 }
 
+#[cfg(feature = "experimental_extras")]
+/// Parses a list of optional dependencies from a string `feat1, feat2, feat3]` -> `vec![feat1, feat2, feat3]`.
+pub fn parse_extras(input: &str) -> Result<Vec<String>, ParseMatchSpecError> {
+    use nom::{
+        combinator::{all_consuming, map},
+        multi::separated_list1,
+    };
+
+    fn parse_feature_name(i: &str) -> IResult<&str, &str> {
+        delimited(
+            multispace0,
+            take_while1(|c: char| c.is_alphanumeric() || c == '_' || c == '-'),
+            multispace0,
+        )(i)
+    }
+
+    fn parse_features(i: &str) -> IResult<&str, Vec<String>> {
+        separated_list1(char(','), map(parse_feature_name, |s: &str| s.to_string()))(i)
+    }
+
+    match all_consuming(parse_features)(input).finish() {
+        Ok((_remaining, features)) => Ok(features),
+        Err(_e) => Err(ParseMatchSpecError::InvalidBracket),
+    }
+}
+
 /// Parses a [`BracketVec`] into precise components
 fn parse_bracket_vec_into_components(
     bracket: BracketVec<'_>,
@@ -248,6 +278,17 @@ fn parse_bracket_vec_into_components(
             "version" => match_spec.version = Some(VersionSpec::from_str(value, strictness)?),
             "build" => match_spec.build = Some(StringMatcher::from_str(value)?),
             "build_number" => match_spec.build_number = Some(BuildNumberSpec::from_str(value)?),
+            "extras" => {
+                // Optional features are still experimental
+                #[cfg(feature = "experimental_extras")]
+                {
+                    match_spec.extras = Some(parse_extras(value)?);
+                }
+                #[cfg(not(feature = "experimental_extras"))]
+                {
+                    return Err(ParseMatchSpecError::InvalidBracketKey("extras".to_string()));
+                }
+            }
             "sha256" => {
                 match_spec.sha256 = Some(
                     parse_digest_from_hex::<Sha256>(value)
@@ -699,6 +740,9 @@ mod tests {
         NamelessMatchSpec, ParseChannelError, ParseStrictness, ParseStrictness::*, VersionSpec,
     };
 
+    #[cfg(feature = "experimental_extras")]
+    use crate::match_spec::parse::parse_extras;
+
     fn channel_config() -> ChannelConfig {
         ChannelConfig::default_with_root_dir(
             std::env::current_dir().expect("Could not get current directory"),
@@ -1342,6 +1386,7 @@ mod tests {
             build: "py27_0*".parse().ok(),
             build_number: Some(BuildNumberSpec::from_str(">=6").unwrap()),
             file_name: Some("foo-1.0-py27_0.tar.bz2".to_string()),
+            extras: None,
             channel: Some(
                 Channel::from_str("conda-forge", &channel_config())
                     .map(Arc::new)
@@ -1375,4 +1420,70 @@ mod tests {
             .collect::<Vec<_>>();
         assert_eq!(specs, parsed_specs);
     }
+
+    #[cfg(feature = "experimental_extras")]
+    #[test]
+    fn test_simple_extras() {
+        let spec = MatchSpec::from_str("foo[extras=[bar]]", Strict).unwrap();
+
+        assert_eq!(spec.extras, Some(vec!["bar".to_string()]));
+        assert!(MatchSpec::from_str("foo[extras=[bar,baz]", Strict).is_err());
+    }
+
+    #[cfg(feature = "experimental_extras")]
+    #[test]
+    fn test_multiple_extras() {
+        let spec = MatchSpec::from_str("foo[extras=[bar,baz]]", Strict).unwrap();
+        assert_eq!(
+            spec.extras,
+            Some(vec!["bar".to_string(), "baz".to_string()])
+        );
+    }
+
+    #[cfg(feature = "experimental_extras")]
+    #[test]
+    fn test_parse_extras() {
+        assert_eq!(
+            parse_extras("bar,baz").unwrap(),
+            vec!["bar".to_string(), "baz".to_string()]
+        );
+        assert_eq!(parse_extras("bar").unwrap(), vec!["bar".to_string()]);
+        assert_eq!(
+            parse_extras("bar, baz").unwrap(),
+            vec!["bar".to_string(), "baz".to_string()]
+        );
+        assert!(parse_extras("[bar,baz]").is_err());
+    }
+
+    #[cfg(feature = "experimental_extras")]
+    #[test]
+    fn test_invalid_extras() {
+        // Empty extras value
+        assert!(MatchSpec::from_str("foo[extras=]", Strict).is_err());
+
+        // Missing brackets around extras list
+        assert!(MatchSpec::from_str("foo[extras=bar,baz]", Strict).is_err());
+
+        // Trailing comma in extras list
+        assert!(MatchSpec::from_str("foo[extras=[bar,]]", Strict).is_err());
+
+        // Invalid characters in extras name
+        assert!(MatchSpec::from_str("foo[extras=[bar!,baz]]", Strict).is_err());
+
+        // Invalid characters in extras name
+        println!(
+            "{:?}",
+            MatchSpec::from_str("foo[extras=[bar!,baz]]", Strict)
+        );
+        assert!(MatchSpec::from_str("foo[extras=[bar!,baz]]", Strict).is_err());
+
+        // Empty extras item
+        assert!(MatchSpec::from_str("foo[extras=[bar,,baz]]", Strict).is_err());
+
+        // Missing closing bracket
+        assert!(MatchSpec::from_str("foo[extras=[bar,baz", Strict).is_err());
+
+        // Missing opening bracket
+        assert!(MatchSpec::from_str("foo[extras=bar,baz]]", Strict).is_err());
+    }
 }

crates/rattler_conda_types/src/repo_data/mod.rs

@@ -6,7 +6,7 @@ pub mod sharded;
 mod topological_sort;
 
 use std::{
-    collections::BTreeSet,
+    collections::{BTreeMap, BTreeSet},
     fmt::{Display, Formatter},
     path::Path,
 };
@@ -114,6 +114,11 @@ pub struct PackageRecord {
     #[serde(default)]
     pub depends: Vec<String>,
 
+    /// Specifications of optional or dependencies. These are dependencies that are
+    /// only required if certain features are enabled or if certain conditions are met.
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub extra_depends: BTreeMap<String, Vec<String>>,
+
     /// Features are a deprecated way to specify different feature sets for the
     /// conda solver. This is not supported anymore and should not be used.
     /// Instead, `mutex` packages should be used to specify
@@ -323,6 +328,7 @@ impl PackageRecord {
             noarch: NoArchType::default(),
             platform: None,
             python_site_packages_path: None,
+            extra_depends: BTreeMap::new(),
             sha256: None,
             size: None,
             subdir: Platform::current().to_string(),
@@ -511,6 +517,7 @@ impl PackageRecord {
             noarch: index.noarch,
             platform: index.platform,
             python_site_packages_path: index.python_site_packages_path,
+            extra_depends: BTreeMap::new(),
             sha256,
             size,
             subdir,