From 4d7ffeaf4b9f95419c7920841bd6fff5451e7e8b Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Wed, 17 Apr 2024 21:58:59 +0200 Subject: [PATCH 01/13] fix: github clone with ssh --- crates/pop-parachains/src/new_parachain.rs | 13 +++---- crates/pop-parachains/src/utils/git.rs | 41 ++++++++++++++++++++-- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/crates/pop-parachains/src/new_parachain.rs b/crates/pop-parachains/src/new_parachain.rs index f38245a5..a977d3a8 100644 --- a/crates/pop-parachains/src/new_parachain.rs +++ b/crates/pop-parachains/src/new_parachain.rs @@ -29,15 +29,10 @@ pub fn instantiate_template_dir( config: Config, ) -> Result> { sanitize(target)?; - use Template::*; - let url = match template { - FPT => "https://github.com/paritytech/frontier-parachain-template.git", - Contracts => "https://github.com/paritytech/substrate-contracts-node.git", - Base => { - return instantiate_base_template(target, config); - }, + if matches!(template, Template::Base) { + return instantiate_base_template(target, config); }; - let tag = Git::clone_and_degit(url, target)?; + let tag = Git::clone_and_degit(template, target)?; Repository::init(target)?; Ok(tag) } @@ -45,7 +40,7 @@ pub fn instantiate_template_dir( pub fn instantiate_base_template(target: &Path, config: Config) -> Result> { let temp_dir = ::tempfile::TempDir::new_in(std::env::temp_dir())?; let source = temp_dir.path(); - let tag = Git::clone_and_degit("https://github.com/r0gue-io/base-parachain", source)?; + let tag = Git::clone_and_degit(&crate::Template::Base, source)?; for entry in WalkDir::new(&source) { let entry = entry?; diff --git a/crates/pop-parachains/src/utils/git.rs b/crates/pop-parachains/src/utils/git.rs index 264ae487..1c797fc9 100644 --- a/crates/pop-parachains/src/utils/git.rs +++ b/crates/pop-parachains/src/utils/git.rs @@ -1,10 +1,13 @@ use anyhow::{anyhow, Result}; use git2::{build::RepoBuilder, FetchOptions, IndexAddOption, Repository, ResetType}; +use git2::{Cred, RemoteCallbacks}; use regex::Regex; -use std::fs; use std::path::Path; +use std::{env, fs}; use url::Url; +use crate::Template; + pub struct Git; impl Git { pub(crate) fn clone(url: &Url, working_dir: &Path, branch: Option<&str>) -> Result<()> { @@ -21,8 +24,13 @@ impl Git { Ok(()) } /// Clone `url` into `target` and degit it - pub fn clone_and_degit(url: &str, target: &Path) -> Result> { - let repo = Repository::clone(url, target)?; + pub fn clone_and_degit(template: &Template, target: &Path) -> Result> { + let url = match template { + Template::FPT => "https://github.com/paritytech/frontier-parachain-template.git", + Template::Contracts => "https://github.com/paritytech/substrate-contracts-node.git", + Template::Base => "https://github.com/r0gue-io/base-parachain", + }; + let repo = Repository::clone(url, target).unwrap_or(Self::ssh_clone(template, target)?); // fetch tags from remote let release = Self::fetch_latest_tag(&repo); @@ -32,6 +40,33 @@ impl Git { Ok(release) } + /// For users that have ssh configuration for cloning repositories + fn ssh_clone(template: &Template, target: &Path) -> Result { + let ssh_url = match template { + Template::FPT => "git@github.com:paritytech/frontier-parachain-template.git", + Template::Contracts => "git@github.com:paritytech/substrate-contracts-node.git", + Template::Base => "git@github.com:r0gue-io/base-parachain.git", + }; + // Prepare callback and fetch options. + let mut callbacks = RemoteCallbacks::new(); + callbacks.credentials(|_url, username_from_url, _allowed_types| { + Cred::ssh_key( + username_from_url.unwrap(), + None, + std::path::Path::new(&format!("{}/.ssh/id_rsa", env::var("HOME").unwrap())), + None, + ) + }); + let mut fo = git2::FetchOptions::new(); + fo.remote_callbacks(callbacks); + + // Prepare builder and clone. + let mut builder = git2::build::RepoBuilder::new(); + builder.fetch_options(fo); + let repo = builder.clone(ssh_url, target)?; + Ok(repo) + } + /// Fetch the latest release from a repository fn fetch_latest_tag(repo: &Repository) -> Option { let version_reg = Regex::new(r"v\d+\.\d+\.\d+").expect("Valid regex"); From 94ca3a4718c90d8df506b53ba67761762f101ee9 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Thu, 18 Apr 2024 21:51:02 +0200 Subject: [PATCH 02/13] fix: ssh keys can not be in HOME --- crates/pop-parachains/src/utils/git.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/pop-parachains/src/utils/git.rs b/crates/pop-parachains/src/utils/git.rs index 1c797fc9..d6fb8e83 100644 --- a/crates/pop-parachains/src/utils/git.rs +++ b/crates/pop-parachains/src/utils/git.rs @@ -30,6 +30,7 @@ impl Git { Template::Contracts => "https://github.com/paritytech/substrate-contracts-node.git", Template::Base => "https://github.com/r0gue-io/base-parachain", }; + println!("before"); let repo = Repository::clone(url, target).unwrap_or(Self::ssh_clone(template, target)?); // fetch tags from remote @@ -42,6 +43,7 @@ impl Git { /// For users that have ssh configuration for cloning repositories fn ssh_clone(template: &Template, target: &Path) -> Result { + println!("sshs"); let ssh_url = match template { Template::FPT => "git@github.com:paritytech/frontier-parachain-template.git", Template::Contracts => "git@github.com:paritytech/substrate-contracts-node.git", @@ -49,13 +51,19 @@ impl Git { }; // Prepare callback and fetch options. let mut callbacks = RemoteCallbacks::new(); - callbacks.credentials(|_url, username_from_url, _allowed_types| { - Cred::ssh_key( - username_from_url.unwrap(), - None, - std::path::Path::new(&format!("{}/.ssh/id_rsa", env::var("HOME").unwrap())), - None, - ) + callbacks.credentials(|_url, username_from_url, allowed_types| { + if allowed_types.contains(git2::CredentialType::SSH_KEY) { + if std::env::var("SSH_AGENT_PID").is_ok() { + return git2::Cred::ssh_key_from_agent(username_from_url.unwrap()); + } + if let Ok(home_dir) = std::env::var("HOME") { + let key_path = std::path::Path::new(&home_dir).join(".ssh").join("id_rsa"); + if key_path.is_file() { + return git2::Cred::ssh_key(username_from_url.unwrap(), None, &key_path, None); + } + } + } + git2::Cred::default() }); let mut fo = git2::FetchOptions::new(); fo.remote_callbacks(callbacks); From 2fb5500eee0adae2afba91346b4f0769d0ad95bd Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Fri, 19 Apr 2024 11:40:14 +0200 Subject: [PATCH 03/13] chore: use git2_credentials for ssh auth --- Cargo.lock | 33 +++++++++++++++++ Cargo.toml | 1 + crates/pop-cli/src/commands/new/parachain.rs | 4 +- crates/pop-parachains/Cargo.toml | 1 + crates/pop-parachains/src/new_parachain.rs | 5 +-- crates/pop-parachains/src/utils/git.rs | 39 +++++++++++--------- 6 files changed, 62 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c6275c6..bb05d9ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2233,6 +2233,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dialoguer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59c6f2989294b9a498d3ad5491a79c6deb604617378e1cdc4bfc1c1361fe2f87" +dependencies = [ + "console", + "shell-words", + "tempfile", + "zeroize", +] + [[package]] name = "difflib" version = "0.4.0" @@ -3045,6 +3057,20 @@ dependencies = [ "url", ] +[[package]] +name = "git2_credentials" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a297fe29addafaf3c774fdbb8f410e487b819555f067ed94c44c5c2ae15e3702" +dependencies = [ + "dialoguer", + "dirs", + "git2", + "pest", + "pest_derive", + "regex", +] + [[package]] name = "group" version = "0.13.0" @@ -5474,6 +5500,7 @@ dependencies = [ "askama", "duct", "git2", + "git2_credentials", "indexmap 2.2.6", "regex", "reqwest", @@ -6852,6 +6879,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "signal-hook" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 34e9ff08..2615a10d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ serde_json = { version = "1.0"} serde = { version = "1.0", features = ["derive"] } zombienet-sdk = { git = "https://github.com/r0gue-io/zombienet-sdk", branch = "pop", version = "0.1.0-alpha.1"} zombienet-support = { git = "https://github.com/r0gue-io/zombienet-sdk", branch = "pop", version = "0.1.0-alpha.1" } +git2_credentials = "0.13.0" # pop-cli clap = { version = "4.4", features = ["derive"] } diff --git a/crates/pop-cli/src/commands/new/parachain.rs b/crates/pop-cli/src/commands/new/parachain.rs index c86aaac7..7def296c 100644 --- a/crates/pop-cli/src/commands/new/parachain.rs +++ b/crates/pop-cli/src/commands/new/parachain.rs @@ -94,12 +94,14 @@ impl NewParachainCommand { initial_endowment: self.initial_endowment.clone().expect("default values"), }, )?; + spinner.stop("Generation complete"); + spinner.start("Git init"); if let Err(err) = Git::git_init(destination_path.as_path(), "initialized parachain") { if err.class() == git2::ErrorClass::Config && err.code() == git2::ErrorCode::NotFound { outro_cancel("git signature could not be found. Please configure your git config with your name and email")?; } } - spinner.stop("Generation complete"); + spinner.stop("Git init complete"); if let Some(tag) = tag { log::info(format!("Version: {}", tag))?; } diff --git a/crates/pop-parachains/Cargo.toml b/crates/pop-parachains/Cargo.toml index cf7104f3..e17c5b19 100644 --- a/crates/pop-parachains/Cargo.toml +++ b/crates/pop-parachains/Cargo.toml @@ -10,6 +10,7 @@ edition = "2021" anyhow.workspace = true duct.workspace = true git2.workspace = true +git2_credentials.workspace = true tempfile.workspace = true url.workspace = true tokio.workspace = true diff --git a/crates/pop-parachains/src/new_parachain.rs b/crates/pop-parachains/src/new_parachain.rs index a977d3a8..d1157b76 100644 --- a/crates/pop-parachains/src/new_parachain.rs +++ b/crates/pop-parachains/src/new_parachain.rs @@ -6,7 +6,6 @@ use crate::{ }, }; use anyhow::Result; -use git2::Repository; use std::{fs, path::Path}; use walkdir::WalkDir; @@ -33,7 +32,7 @@ pub fn instantiate_template_dir( return instantiate_base_template(target, config); }; let tag = Git::clone_and_degit(template, target)?; - Repository::init(target)?; + //Repository::init(target)?; Ok(tag) } @@ -67,7 +66,7 @@ pub fn instantiate_base_template(target: &Path, config: Config) -> Result