From 85d9547237943ffa6d2d96644a9d597e7be1043f Mon Sep 17 00:00:00 2001 From: Bela Stoyan Date: Wed, 17 Jan 2024 07:56:53 +0100 Subject: [PATCH] feat: add conda-forge integration (#465) --- rust-tests/src/lib.rs | 36 +++++ src/main.rs | 57 ++++++++ src/upload/conda_forge.rs | 157 ++++++++++++++++++++++ src/upload/mod.rs | 10 +- src/upload/package.rs | 6 + test-data/recipes/polarify/linux_64_.yaml | 8 ++ test-data/recipes/polarify/recipe.yaml | 39 ++++++ 7 files changed, 308 insertions(+), 5 deletions(-) create mode 100644 src/upload/conda_forge.rs create mode 100644 test-data/recipes/polarify/linux_64_.yaml create mode 100644 test-data/recipes/polarify/recipe.yaml diff --git a/rust-tests/src/lib.rs b/rust-tests/src/lib.rs index 5f7249875..85383ea09 100644 --- a/rust-tests/src/lib.rs +++ b/rust-tests/src/lib.rs @@ -489,6 +489,42 @@ mod tests { assert!(rattler_build.unwrap().status.success()); } + #[test] + fn test_dry_run_cf_upload() { + let tmp = tmp("test_polarify"); + let variant = recipes().join("polarify").join("linux_64_.yaml"); + let rattler_build = rattler().build::<_, _, PathBuf>( + recipes().join("polarify"), + tmp.as_dir(), + Some(variant), + ); + + assert!(rattler_build.is_ok()); + assert!(rattler_build.unwrap().status.success()); + + // try to upload the package using the rattler upload command + let pkg_path = get_package(tmp.as_dir(), "polarify".to_string()); + let rattler_upload = rattler() + .with_args([ + "upload", + "-vvv", + "conda-forge", + "--feedstock", + "polarify", + "--feedstock-token", + "fake-feedstock-token", + "--staging-token", + "fake-staging-token", + "--dry-run", + pkg_path.to_str().unwrap(), + ]) + .expect("failed to run rattler upload"); + + let output = String::from_utf8(rattler_upload.stderr).unwrap(); + assert!(rattler_upload.status.success()); + assert!(output.contains("Done uploading packages to conda-forge")); + } + #[test] fn test_correct_sha256() { let tmp = tmp("correct-sha"); diff --git a/src/main.rs b/src/main.rs index c7195b6a7..1036acb38 100644 --- a/src/main.rs +++ b/src/main.rs @@ -267,6 +267,8 @@ enum ServerType { Artifactory(ArtifactoryOpts), Prefix(PrefixOpts), Anaconda(AnacondaOpts), + #[clap(hide = true)] + CondaForge(CondaForgeOpts), } #[derive(Clone, Debug, PartialEq, Parser)] @@ -358,6 +360,54 @@ struct AnacondaOpts { force: bool, } +/// Options for uploading to conda-forge +#[derive(Clone, Debug, PartialEq, Parser)] +pub struct CondaForgeOpts { + /// The Anaconda API key + #[arg(long, env = "STAGING_BINSTAR_TOKEN", required = true)] + staging_token: String, + + /// The feedstock name + #[arg(long, env = "FEEDSTOCK_NAME", required = true)] + feedstock: String, + + /// The feedstock token + #[arg(long, env = "FEEDSTOCK_TOKEN", required = true)] + feedstock_token: String, + + /// The staging channel name + #[arg(long, env = "STAGING_CHANNEL", default_value = "cf-staging")] + staging_channel: String, + + /// The Anaconda Server URL + #[arg( + long, + env = "ANACONDA_SERVER_URL", + default_value = "https://api.anaconda.org" + )] + anaconda_url: Url, + + /// The validation endpoint url + #[arg( + long, + env = "VALIDATION_ENDPOINT", + default_value = "https://conda-forge.herokuapp.com/feedstock-outputs/copy" + )] + validation_endpoint: Url, + + /// Post comment on promotion failure + #[arg(long, env = "POST_COMMENT_ON_ERROR", default_value = "true")] + post_comment_on_error: bool, + + /// The CI provider + #[arg(long, env = "CI")] + provider: Option, + + /// Dry run, don't actually upload anything + #[arg(long, env = "DRY_RUN", default_value = "false")] + dry_run: bool, +} + #[tokio::main] async fn main() -> miette::Result<()> { let args = App::parse(); @@ -780,6 +830,13 @@ async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { ) .await?; } + ServerType::CondaForge(conda_forge_opts) => { + upload::conda_forge::upload_packages_to_conda_forge( + conda_forge_opts, + &args.package_files, + ) + .await?; + } } Ok(()) diff --git a/src/upload/conda_forge.rs b/src/upload/conda_forge.rs new file mode 100644 index 000000000..ee247e9aa --- /dev/null +++ b/src/upload/conda_forge.rs @@ -0,0 +1,157 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use miette::{miette, IntoDiagnostic}; +use tracing::{debug, info}; + +use crate::{upload::get_default_client, CondaForgeOpts}; + +use super::{ + anaconda, + package::{self}, +}; + +async fn get_channel_target_from_variant_config( + variant_config_path: &Path, +) -> miette::Result { + let variant_config = tokio::fs::read_to_string(variant_config_path) + .await + .into_diagnostic()?; + + let variant_config: serde_yaml::Value = + serde_yaml::from_str(&variant_config).into_diagnostic()?; + + let channel_target = variant_config + .get("channel_targets") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + miette!("\"channel_targets\" not found or invalid format in variant_config") + })?; + + let (channel, label) = channel_target + .split_once(' ') + .ok_or_else(|| miette!("Invalid channel_target format"))?; + + if channel != "conda-forge" { + return Err(miette!("channel_target is not a conda-forge channel")); + } + + Ok(label.to_string()) +} + +pub async fn upload_packages_to_conda_forge( + opts: CondaForgeOpts, + package_files: &Vec, +) -> miette::Result<()> { + let anaconda = anaconda::Anaconda::new(opts.staging_token, opts.anaconda_url); + + let mut channels: HashMap> = HashMap::new(); + + for package_file in package_files { + let package = package::ExtractedPackage::from_package_file(package_file)?; + + let variant_config_path = package + .extraction_dir() + .join("info") + .join("recipe") + .join("variant_config.yaml"); + + let channel = get_channel_target_from_variant_config(&variant_config_path) + .await + .map_err(|e| { + miette!( + "Failed to get channel_targets from variant config for {}: {}", + package.path().display(), + e + ) + })?; + + if !opts.dry_run { + anaconda + .create_or_update_package(&opts.staging_channel, &package) + .await?; + + anaconda + .create_or_update_release(&opts.staging_channel, &package) + .await?; + + anaconda + .upload_file(&opts.staging_channel, &[channel.clone()], false, &package) + .await?; + } else { + debug!( + "Would have uploaded {} to anaconda.org {}/{}", + package.path().display(), + opts.staging_channel, + channel + ); + }; + + let dist_name = format!( + "{}/{}", + package.subdir().ok_or(miette::miette!("No subdir found"))?, + package + .filename() + .ok_or(miette::miette!("No filename found"))? + ); + + channels + .entry(channel) + .or_default() + .insert(dist_name, package.sha256().into_diagnostic()?); + } + + for (channel, checksums) in channels { + info!("Uploading packages for conda-forge channel {}", channel); + + let payload = serde_json::json!({ + "feedstock": opts.feedstock, + "outputs": checksums, + "channel": channel, + "comment_on_error": opts.post_comment_on_error, + "hash_type": "sha256", + "provider": opts.provider + }); + + let client = get_default_client().into_diagnostic()?; + + debug!( + "Sending payload to validation endpoint: {}", + serde_json::to_string_pretty(&payload).into_diagnostic()? + ); + + if opts.dry_run { + debug!( + "Would have sent payload to validation endpoint {}", + opts.validation_endpoint + ); + + continue; + } + + let resp = client + .post(opts.validation_endpoint.clone()) + .json(&payload) + .header("FEEDSTOCK_TOKEN", opts.feedstock_token.clone()) + .send() + .await + .into_diagnostic()?; + + let status = resp.status(); + + let body: serde_json::Value = resp.json().await.into_diagnostic()?; + + debug!( + "Copying to conda-forge/{} returned status code {} with body: {}", + channel, + status, + serde_json::to_string_pretty(&body).into_diagnostic()? + ); + } + + info!("Done uploading packages to conda-forge"); + + Ok(()) +} diff --git a/src/upload/mod.rs b/src/upload/mod.rs index 0a6dd659c..44728a4aa 100644 --- a/src/upload/mod.rs +++ b/src/upload/mod.rs @@ -15,6 +15,7 @@ use url::Url; use crate::upload::package::{sha256_sum, ExtractedPackage}; mod anaconda; +pub mod conda_forge; mod package; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -38,7 +39,7 @@ fn default_bytes_style() -> Result { )) } -fn get_client() -> Result { +fn get_default_client() -> Result { reqwest::Client::builder() .no_gzip() .user_agent(format!("rattler-build/{}", VERSION)) @@ -73,7 +74,7 @@ pub async fn upload_package_to_quetz( }, }; - let client = get_client().into_diagnostic()?; + let client = get_default_client().into_diagnostic()?; for package_file in package_files { let upload_url = url @@ -146,7 +147,7 @@ pub async fn upload_package_to_artifactory( package_file.display() ))?; - let client = get_client().into_diagnostic()?; + let client = get_default_client().into_diagnostic()?; let upload_url = url .join(&format!("{}/{}/{}", channel, subdir, package_name)) @@ -205,7 +206,7 @@ pub async fn upload_package_to_prefix( .join(&format!("api/v1/upload/{}", channel)) .into_diagnostic()?; - let client = get_client().into_diagnostic()?; + let client = get_default_client().into_diagnostic()?; let hash = sha256_sum(package_file).into_diagnostic()?; @@ -234,7 +235,6 @@ pub async fn upload_package_to_anaconda( channels: Vec, force: bool, ) -> miette::Result<()> { - println!("{:?}", channels); let token = match token { Some(token) => token, None => match storage.get("anaconda.org") { diff --git a/src/upload/package.rs b/src/upload/package.rs index d1cab9b8d..d49a2efe4 100644 --- a/src/upload/package.rs +++ b/src/upload/package.rs @@ -20,6 +20,7 @@ pub struct ExtractedPackage<'a> { file: &'a Path, about_json: AboutJson, index_json: IndexJson, + extraction_dir: tempfile::TempDir, } impl<'a> ExtractedPackage<'a> { @@ -38,6 +39,7 @@ impl<'a> ExtractedPackage<'a> { file, about_json, index_json, + extraction_dir, }) } @@ -81,4 +83,8 @@ impl<'a> ExtractedPackage<'a> { pub fn index_json(&self) -> &IndexJson { &self.index_json } + + pub fn extraction_dir(&self) -> &Path { + self.extraction_dir.path() + } } diff --git a/test-data/recipes/polarify/linux_64_.yaml b/test-data/recipes/polarify/linux_64_.yaml new file mode 100644 index 000000000..1eb38c093 --- /dev/null +++ b/test-data/recipes/polarify/linux_64_.yaml @@ -0,0 +1,8 @@ +cdt_name: +- cos6 +channel_sources: +- conda-forge +channel_targets: +- conda-forge polarify-rattler-build_dev +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 diff --git a/test-data/recipes/polarify/recipe.yaml b/test-data/recipes/polarify/recipe.yaml new file mode 100644 index 000000000..af2700630 --- /dev/null +++ b/test-data/recipes/polarify/recipe.yaml @@ -0,0 +1,39 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json + +context: + name: polarify + version: "0.1.3" + +package: + name: ${{ name | lower }} + version: ${{ version }} + +source: + url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/polarify-${{ version }}.tar.gz + sha256: a172a25c73936f23448d3d9a29bfbd578ac4e676962a78e546da8be86e14745d + +build: + number: 3 + noarch: python + script: + - python -m pip install . -vv + +requirements: + host: + - python >=3.9 + - pip + - hatchling + run: + - python >=3.9 + - polars >=0.14.24,<0.20 + +about: + homepage: https://github.com/quantco/polarify + summary: Simplifying conditional Polars Expressions with Python 🐍 🐻‍❄️ + license: BSD-3-Clause + license_file: LICENSE + +extra: + recipe-maintainers: + - pavelzw + - '0xbe7a'