feat: add conda-forge integration (#465)

rust-tests/src/lib.rs

@@ -489,6 +489,42 @@ mod tests {
         assert!(rattler_build.unwrap().status.success());
     }
 
+    #[test]
+    fn test_dry_run_cf_upload() {
+        let tmp = tmp("test_polarify");
+        let variant = recipes().join("polarify").join("linux_64_.yaml");
+        let rattler_build = rattler().build::<_, _, PathBuf>(
+            recipes().join("polarify"),
+            tmp.as_dir(),
+            Some(variant),
+        );
+
+        assert!(rattler_build.is_ok());
+        assert!(rattler_build.unwrap().status.success());
+
+        // try to upload the package using the rattler upload command
+        let pkg_path = get_package(tmp.as_dir(), "polarify".to_string());
+        let rattler_upload = rattler()
+            .with_args([
+                "upload",
+                "-vvv",
+                "conda-forge",
+                "--feedstock",
+                "polarify",
+                "--feedstock-token",
+                "fake-feedstock-token",
+                "--staging-token",
+                "fake-staging-token",
+                "--dry-run",
+                pkg_path.to_str().unwrap(),
+            ])
+            .expect("failed to run rattler upload");
+
+        let output = String::from_utf8(rattler_upload.stderr).unwrap();
+        assert!(rattler_upload.status.success());
+        assert!(output.contains("Done uploading packages to conda-forge"));
+    }
+
     #[test]
     fn test_correct_sha256() {
         let tmp = tmp("correct-sha");

src/main.rs

@@ -267,6 +267,8 @@ enum ServerType {
     Artifactory(ArtifactoryOpts),
     Prefix(PrefixOpts),
     Anaconda(AnacondaOpts),
+    #[clap(hide = true)]
+    CondaForge(CondaForgeOpts),
 }
 
 #[derive(Clone, Debug, PartialEq, Parser)]
@@ -358,6 +360,54 @@ struct AnacondaOpts {
     force: bool,
 }
 
+/// Options for uploading to conda-forge
+#[derive(Clone, Debug, PartialEq, Parser)]
+pub struct CondaForgeOpts {
+    /// The Anaconda API key
+    #[arg(long, env = "STAGING_BINSTAR_TOKEN", required = true)]
+    staging_token: String,
+
+    /// The feedstock name
+    #[arg(long, env = "FEEDSTOCK_NAME", required = true)]
+    feedstock: String,
+
+    /// The feedstock token
+    #[arg(long, env = "FEEDSTOCK_TOKEN", required = true)]
+    feedstock_token: String,
+
+    /// The staging channel name
+    #[arg(long, env = "STAGING_CHANNEL", default_value = "cf-staging")]
+    staging_channel: String,
+
+    /// The Anaconda Server URL
+    #[arg(
+        long,
+        env = "ANACONDA_SERVER_URL",
+        default_value = "https://api.anaconda.org"
+    )]
+    anaconda_url: Url,
+
+    /// The validation endpoint url
+    #[arg(
+        long,
+        env = "VALIDATION_ENDPOINT",
+        default_value = "https://conda-forge.herokuapp.com/feedstock-outputs/copy"
+    )]
+    validation_endpoint: Url,
+
+    /// Post comment on promotion failure
+    #[arg(long, env = "POST_COMMENT_ON_ERROR", default_value = "true")]
+    post_comment_on_error: bool,
+
+    /// The CI provider
+    #[arg(long, env = "CI")]
+    provider: Option<String>,
+
+    /// Dry run, don't actually upload anything
+    #[arg(long, env = "DRY_RUN", default_value = "false")]
+    dry_run: bool,
+}
+
 #[tokio::main]
 async fn main() -> miette::Result<()> {
     let args = App::parse();
@@ -780,6 +830,13 @@ async fn upload_from_args(args: UploadOpts) -> miette::Result<()> {
             )
             .await?;
         }
+        ServerType::CondaForge(conda_forge_opts) => {
+            upload::conda_forge::upload_packages_to_conda_forge(
+                conda_forge_opts,
+                &args.package_files,
+            )
+            .await?;
+        }
     }
 
     Ok(())

src/upload/conda_forge.rs

@@ -0,0 +1,157 @@
+use std::{
+    collections::HashMap,
+    path::{Path, PathBuf},
+};
+
+use miette::{miette, IntoDiagnostic};
+use tracing::{debug, info};
+
+use crate::{upload::get_default_client, CondaForgeOpts};
+
+use super::{
+    anaconda,
+    package::{self},
+};
+
+async fn get_channel_target_from_variant_config(
+    variant_config_path: &Path,
+) -> miette::Result<String> {
+    let variant_config = tokio::fs::read_to_string(variant_config_path)
+        .await
+        .into_diagnostic()?;
+
+    let variant_config: serde_yaml::Value =
+        serde_yaml::from_str(&variant_config).into_diagnostic()?;
+
+    let channel_target = variant_config
+        .get("channel_targets")
+        .and_then(|v| v.as_str())
+        .ok_or_else(|| {
+            miette!("\"channel_targets\" not found or invalid format in variant_config")
+        })?;
+
+    let (channel, label) = channel_target
+        .split_once(' ')
+        .ok_or_else(|| miette!("Invalid channel_target format"))?;
+
+    if channel != "conda-forge" {
+        return Err(miette!("channel_target is not a conda-forge channel"));
+    }
+
+    Ok(label.to_string())
+}
+
+pub async fn upload_packages_to_conda_forge(
+    opts: CondaForgeOpts,
+    package_files: &Vec<PathBuf>,
+) -> miette::Result<()> {
+    let anaconda = anaconda::Anaconda::new(opts.staging_token, opts.anaconda_url);
+
+    let mut channels: HashMap<String, HashMap<_, _>> = HashMap::new();
+
+    for package_file in package_files {
+        let package = package::ExtractedPackage::from_package_file(package_file)?;
+
+        let variant_config_path = package
+            .extraction_dir()
+            .join("info")
+            .join("recipe")
+            .join("variant_config.yaml");
+
+        let channel = get_channel_target_from_variant_config(&variant_config_path)
+            .await
+            .map_err(|e| {
+                miette!(
+                    "Failed to get channel_targets from variant config for {}: {}",
+                    package.path().display(),
+                    e
+                )
+            })?;
+
+        if !opts.dry_run {
+            anaconda
+                .create_or_update_package(&opts.staging_channel, &package)
+                .await?;
+
+            anaconda
+                .create_or_update_release(&opts.staging_channel, &package)
+                .await?;
+
+            anaconda
+                .upload_file(&opts.staging_channel, &[channel.clone()], false, &package)
+                .await?;
+        } else {
+            debug!(
+                "Would have uploaded {} to anaconda.org {}/{}",
+                package.path().display(),
+                opts.staging_channel,
+                channel
+            );
+        };
+
+        let dist_name = format!(
+            "{}/{}",
+            package.subdir().ok_or(miette::miette!("No subdir found"))?,
+            package
+                .filename()
+                .ok_or(miette::miette!("No filename found"))?
+        );
+
+        channels
+            .entry(channel)
+            .or_default()
+            .insert(dist_name, package.sha256().into_diagnostic()?);
+    }
+
+    for (channel, checksums) in channels {
+        info!("Uploading packages for conda-forge channel {}", channel);
+
+        let payload = serde_json::json!({
+            "feedstock": opts.feedstock,
+            "outputs": checksums,
+            "channel": channel,
+            "comment_on_error": opts.post_comment_on_error,
+            "hash_type": "sha256",
+            "provider": opts.provider
+        });
+
+        let client = get_default_client().into_diagnostic()?;
+
+        debug!(
+            "Sending payload to validation endpoint: {}",
+            serde_json::to_string_pretty(&payload).into_diagnostic()?
+        );
+
+        if opts.dry_run {
+            debug!(
+                "Would have sent payload to validation endpoint {}",
+                opts.validation_endpoint
+            );
+
+            continue;
+        }
+
+        let resp = client
+            .post(opts.validation_endpoint.clone())
+            .json(&payload)
+            .header("FEEDSTOCK_TOKEN", opts.feedstock_token.clone())
+            .send()
+            .await
+            .into_diagnostic()?;
+
+        let status = resp.status();
+
+        let body: serde_json::Value = resp.json().await.into_diagnostic()?;
+
+        debug!(
+            "Copying to conda-forge/{} returned status code {} with body: {}",
+            channel,
+            status,
+            serde_json::to_string_pretty(&body).into_diagnostic()?
+        );
+    }
+
+    info!("Done uploading packages to conda-forge");
+
+    Ok(())
+}

src/upload/mod.rs

@@ -15,6 +15,7 @@ use url::Url;
 use crate::upload::package::{sha256_sum, ExtractedPackage};
 
 mod anaconda;
+pub mod conda_forge;
 mod package;
 
 const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -38,7 +39,7 @@ fn default_bytes_style() -> Result<indicatif::ProgressStyle, TemplateError> {
             ))
 }
 
-fn get_client() -> Result<reqwest::Client, reqwest::Error> {
+fn get_default_client() -> Result<reqwest::Client, reqwest::Error> {
     reqwest::Client::builder()
         .no_gzip()
         .user_agent(format!("rattler-build/{}", VERSION))
@@ -73,7 +74,7 @@ pub async fn upload_package_to_quetz(
         },
     };
 
-    let client = get_client().into_diagnostic()?;
+    let client = get_default_client().into_diagnostic()?;
 
     for package_file in package_files {
         let upload_url = url
@@ -146,7 +147,7 @@ pub async fn upload_package_to_artifactory(
             package_file.display()
         ))?;
 
-        let client = get_client().into_diagnostic()?;
+        let client = get_default_client().into_diagnostic()?;
 
         let upload_url = url
             .join(&format!("{}/{}/{}", channel, subdir, package_name))
@@ -205,7 +206,7 @@ pub async fn upload_package_to_prefix(
             .join(&format!("api/v1/upload/{}", channel))
             .into_diagnostic()?;
 
-        let client = get_client().into_diagnostic()?;
+        let client = get_default_client().into_diagnostic()?;
 
         let hash = sha256_sum(package_file).into_diagnostic()?;
 
@@ -234,7 +235,6 @@ pub async fn upload_package_to_anaconda(
     channels: Vec<String>,
     force: bool,
 ) -> miette::Result<()> {
-    println!("{:?}", channels);
     let token = match token {
         Some(token) => token,
         None => match storage.get("anaconda.org") {

src/upload/package.rs

@@ -20,6 +20,7 @@ pub struct ExtractedPackage<'a> {
     file: &'a Path,
     about_json: AboutJson,
     index_json: IndexJson,
+    extraction_dir: tempfile::TempDir,
 }
 
 impl<'a> ExtractedPackage<'a> {
@@ -38,6 +39,7 @@ impl<'a> ExtractedPackage<'a> {
             file,
             about_json,
             index_json,
+            extraction_dir,
         })
     }
 
@@ -81,4 +83,8 @@ impl<'a> ExtractedPackage<'a> {
     pub fn index_json(&self) -> &IndexJson {
         &self.index_json
     }
+
+    pub fn extraction_dir(&self) -> &Path {
+        self.extraction_dir.path()
+    }
 }

test-data/recipes/polarify/linux_64_.yaml

@@ -0,0 +1,8 @@
+cdt_name:
+- cos6
+channel_sources:
+- conda-forge
+channel_targets:
+- conda-forge polarify-rattler-build_dev
+docker_image:
+- quay.io/condaforge/linux-anvil-cos7-x86_64