From 6cd0ef1b039106a112c25fd9a471af10236539b9 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:58:53 +0200 Subject: [PATCH 01/19] source tanssi-node --- crates/pop-parachains/src/up/parachains.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/pop-parachains/src/up/parachains.rs b/crates/pop-parachains/src/up/parachains.rs index 3bea2cba..bae8adec 100644 --- a/crates/pop-parachains/src/up/parachains.rs +++ b/crates/pop-parachains/src/up/parachains.rs @@ -32,6 +32,13 @@ pub(super) enum Parachain { Fallback = "v0.1.0-alpha2" ))] Pop, + /// Swift and effortless deployment of application-specific blockchains. + #[strum(props( + Repository = "https://github.com/r0gue-io/tanssi", + Binary = "tanssi-node", + Fallback = "v0.8.1" + ))] + Tanssi, } impl TryInto for Parachain { @@ -42,7 +49,7 @@ impl TryInto for Parachain { /// * `latest` - If applicable, some specifier used to determine the latest source. fn try_into(&self, tag: Option, latest: Option) -> Result { Ok(match self { - Parachain::System | Parachain::Pop => { + Parachain::System | Parachain::Pop | Parachain::Tanssi => { // Source from GitHub release asset let repo = GitHub::parse(self.repository())?; Source::GitHub(ReleaseArchive { @@ -135,6 +142,7 @@ pub(super) async fn from( source: TryInto::try_into(para, tag, latest)?, cache: cache.to_path_buf(), }; + println!("{:?}", &chain.unwrap()); return Ok(Some(super::Parachain { id, binary, From 36cf396b50905823fac64aff627482260e7ad55b Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 21:07:21 +0200 Subject: [PATCH 02/19] feat: build specs for tanssi pop up --- crates/pop-cli/src/commands/up/parachain.rs | 3 + crates/pop-parachains/src/build.rs | 54 +++++++++ crates/pop-parachains/src/errors.rs | 2 + crates/pop-parachains/src/up/mod.rs | 117 +++++++++++++++++++- crates/pop-parachains/src/up/parachains.rs | 9 +- 5 files changed, 177 insertions(+), 8 deletions(-) diff --git a/crates/pop-cli/src/commands/up/parachain.rs b/crates/pop-cli/src/commands/up/parachain.rs index fe1bbc1a..e24d964b 100644 --- a/crates/pop-cli/src/commands/up/parachain.rs +++ b/crates/pop-cli/src/commands/up/parachain.rs @@ -90,6 +90,9 @@ impl ZombienetCommand { if Self::source_binaries(&mut zombienet, &cache, self.verbose, self.skip_confirm).await? { return Ok(()); } + if zombienet.binaries().any(|b| b.name() == "tanssi-node") { + zombienet.build_specs()?; + } // Finally spawn network and wait for signal to terminate let spinner = cliclack::spinner(); diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 669ed1e8..999ff207 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -121,6 +121,60 @@ pub fn generate_raw_chain_spec( Ok(raw_chain_spec) } +/// Generates a raw chain specification file for a parachain. +/// +/// # Arguments +/// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. +/// * `plain_chain_spec` - Location of the plain chain specification file. +/// * `chain_spec_file_name` - The name of the chain specification file to be generated. +pub fn generate_raw_chain_spec_tanssi( + binary_path: &Path, + chain: Option<&str>, + raw_chain_spec: &Path, + container_chains: Vec<&str>, + invulnerables: Vec<&str>, + parachain_id: &str, +) -> Result { + check_command_exists(&binary_path, "build-spec")?; + let mut args = vec!["build-spec", "--parachain-id", parachain_id]; + if let Some(chain) = chain { + args.push("--chain"); + args.push(chain); + } + for container_chain in container_chains { + args.push("--add-container-chain"); + args.push(container_chain); + } + for invulnerable in invulnerables { + args.push("--invulnerable"); + args.push(invulnerable); + } + cmd(binary_path, args).stdout_path(raw_chain_spec).stderr_null().run()?; + Ok(raw_chain_spec.to_path_buf()) +} + +/// Generates a raw chain specification file for a parachain. +/// +/// # Arguments +/// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. +/// * `plain_chain_spec` - Location of the plain chain specification file. +/// * `chain_spec_file_name` - The name of the chain specification file to be generated. +pub fn generate_raw_chain_spec_container_chain( + binary_path: &Path, + raw_chain_spec: &Path, + parachain_id: &str, +) -> Result { + check_command_exists(&binary_path, "build-spec")?; + cmd( + binary_path, + vec!["build-spec", "--disable-default-bootnode", "--parachain-id", parachain_id, "--raw"], + ) + .stderr_null() + .stdout_path(&raw_chain_spec) + .run()?; + Ok(raw_chain_spec.to_path_buf()) +} + /// Export the WebAssembly runtime for the parachain. /// /// # Arguments diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index 58b59c58..c3376e7a 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -9,6 +9,8 @@ pub enum Error { Aborted, #[error("Anyhow error: {0}")] AnyhowError(#[from] anyhow::Error), + #[error("Missing chain with id: {0}")] + ChainNotFound(String), #[error("{0}")] CommonError(#[from] pop_common::Error), #[error("Configuration error: {0}")] diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 390e990a..60df6225 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-3.0 -use crate::errors::Error; +use crate::{ + build::{generate_raw_chain_spec_container_chain, generate_raw_chain_spec_tanssi}, + errors::Error, +}; use glob::glob; use indexmap::IndexMap; pub use pop_common::{ @@ -100,6 +103,44 @@ impl Zombienet { .filter_map(|b| b) } + /// Build the needed specs before spawn the network. + pub fn build_specs(&mut self) -> Result<(), Error> { + // This order is important, first container chains that can be more than one, then tanssi-node + let mut container_chains: Vec = Vec::new(); + self.parachains.iter().filter(|(&id, _)| id > 1000).try_for_each( + |(id, parachain)| -> Result<(), Error> { + container_chains.push( + generate_raw_chain_spec_container_chain( + parachain.binary.path().as_path(), + parachain + .chain_spec_path + .as_deref() + .unwrap_or(Path::new("./chain-spec.json")), + &id.to_string(), + )? + .display() + .to_string(), + ); + Ok(()) + }, + )?; + let tanssi_node = self.parachains.get(&1000).ok_or(Error::ChainNotFound("1000".into()))?; + generate_raw_chain_spec_tanssi( + tanssi_node.binary.path().as_path(), + tanssi_node.chain.as_deref(), + tanssi_node.chain_spec_path.as_deref().unwrap_or(Path::new("./chain-spec.json")), + container_chains.iter().map(|s| s.as_str()).collect(), + tanssi_node + .collators_names + .iter() + .filter(|s| s.starts_with("Collator")) + .map(|s| s.as_str()) + .collect(), + &1000.to_string(), + )?; + Ok(()) + } + /// Determine parachain configuration based on specified version and network configuration. /// /// # Arguments @@ -132,7 +173,7 @@ impl Zombienet { as u32; let chain = table.get("chain").and_then(|i| i.as_str()); - + let chain_spec_path = table.get("chain_spec_path").and_then(|i| i.as_str()); let command = NetworkConfiguration::default_command(table) .cloned() .or_else(|| { @@ -175,6 +216,18 @@ impl Zombienet { continue; } + let mut collators_names: Vec<&str> = Vec::new(); + if let Some(collators) = table.get("collators").and_then(|p| p.as_array_of_tables()) { + for collator in collators.iter() { + if let Some(name) = + NetworkConfiguration::name(collator).and_then(|i| i.as_str()) + { + //let n = Some(Item::Value(Value::String(Formatted::new(name.into())))); + collators_names.push(name); + } + } + } + // Check if known parachain let version = parachains.as_ref().and_then(|r| { r.iter() @@ -182,7 +235,17 @@ impl Zombienet { .nth(0) .map(|v| v.as_str()) }); - if let Some(parachain) = parachains::from(id, &command, version, chain, cache).await? { + if let Some(parachain) = parachains::from( + id, + &command, + version, + chain, + cache, + chain_spec_path, + collators_names.clone(), + ) + .await? + { paras.insert(id, parachain); continue; } @@ -190,14 +253,33 @@ impl Zombienet { // Check if parachain binary source specified as an argument if let Some(parachains) = parachains.as_ref() { for repo in parachains.iter().filter(|r| command == r.package) { - paras.insert(id, Parachain::from_repository(id, repo, chain, cache)?); + paras.insert( + id, + Parachain::from_repository( + id, + repo, + chain, + cache, + chain_spec_path, + collators_names, + )?, + ); continue 'outer; } } // Check if command references a local binary if ["./", "../", "/"].iter().any(|p| command.starts_with(p)) { - paras.insert(id, Parachain::from_local(id, command.into(), chain)?); + paras.insert( + id, + Parachain::from_local( + id, + command.into(), + chain, + chain_spec_path, + collators_names, + )?, + ); continue; } @@ -369,6 +451,11 @@ impl NetworkConfiguration { config.get("default_command") } + /// Returns the `name` configuration. + fn name(config: &Table) -> Option<&Item> { + config.get("name") + } + /// Returns the `nodes` configuration. fn nodes(relay_chain: &Table) -> Option<&ArrayOfTables> { relay_chain.get("nodes").and_then(|i| i.as_array_of_tables()) @@ -507,6 +594,10 @@ struct Parachain { chain: Option, /// If applicable, the binary used to generate a chain specification. chain_spec_generator: Option, + /// If applicable, the path to the chain specification. + chain_spec_path: Option, + /// List of collators names. + collators_names: Vec, } impl Parachain { @@ -516,7 +607,13 @@ impl Parachain { /// * `id` - The parachain identifier on the local network. /// * `path` - The path to the local binary. /// * `chain` - The chain specified. - fn from_local(id: u32, path: PathBuf, chain: Option<&str>) -> Result { + fn from_local( + id: u32, + path: PathBuf, + chain: Option<&str>, + chain_spec_path: Option<&str>, + collators_names: Vec<&str>, + ) -> Result { let name = path .file_name() .and_then(|f| f.to_str()) @@ -529,6 +626,8 @@ impl Parachain { binary: Binary::Local { name, path, manifest }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), }) } @@ -545,6 +644,8 @@ impl Parachain { repo: &Repository, chain: Option<&str>, cache: &Path, + chain_spec_path: Option<&str>, + collators_names: Vec<&str>, ) -> Result { // Check for GitHub repository to be able to download source as an archive if repo.url.host_str().is_some_and(|h| h.to_lowercase() == "github.com") { @@ -566,6 +667,8 @@ impl Parachain { }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), }) } else { Ok(Parachain { @@ -583,6 +686,8 @@ impl Parachain { }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), }) } } diff --git a/crates/pop-parachains/src/up/parachains.rs b/crates/pop-parachains/src/up/parachains.rs index bae8adec..e63c3782 100644 --- a/crates/pop-parachains/src/up/parachains.rs +++ b/crates/pop-parachains/src/up/parachains.rs @@ -9,7 +9,7 @@ use pop_common::{ }, target, Error, GitHub, }; -use std::path::Path; +use std::path::{Path, PathBuf}; use strum::VariantArray as _; use strum_macros::{EnumProperty, VariantArray}; @@ -111,6 +111,8 @@ pub(super) async fn system( binary, chain: chain.map(|c| c.to_string()), chain_spec_generator, + chain_spec_path: None, + collators_names: Vec::new(), })); } @@ -128,6 +130,8 @@ pub(super) async fn from( version: Option<&str>, chain: Option<&str>, cache: &Path, + chain_spec_path: Option<&str>, + collators_names: Vec<&str>, ) -> Result, Error> { for para in Parachain::VARIANTS.iter().filter(|p| p.binary() == command) { let releases = para.releases().await?; @@ -142,12 +146,13 @@ pub(super) async fn from( source: TryInto::try_into(para, tag, latest)?, cache: cache.to_path_buf(), }; - println!("{:?}", &chain.unwrap()); return Ok(Some(super::Parachain { id, binary, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), })); } Ok(None) From cc2824ce2ee09e9abfa6a361685a2ac2c9ef302c Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 22:29:46 +0200 Subject: [PATCH 03/19] refactor: build specs container chains --- crates/pop-cli/src/commands/up/parachain.rs | 4 +- crates/pop-parachains/src/build.rs | 65 ++++++++------------- crates/pop-parachains/src/errors.rs | 2 +- crates/pop-parachains/src/up/mod.rs | 49 ++++++++++------ 4 files changed, 59 insertions(+), 61 deletions(-) diff --git a/crates/pop-cli/src/commands/up/parachain.rs b/crates/pop-cli/src/commands/up/parachain.rs index e24d964b..b1da974d 100644 --- a/crates/pop-cli/src/commands/up/parachain.rs +++ b/crates/pop-cli/src/commands/up/parachain.rs @@ -90,8 +90,10 @@ impl ZombienetCommand { if Self::source_binaries(&mut zombienet, &cache, self.verbose, self.skip_confirm).await? { return Ok(()); } + // For container chains, build the specs before spawn the network + // TODO: Is there a better way to identifiy it than to check if there is a hardcoded tanssi-node if zombienet.binaries().any(|b| b.name() == "tanssi-node") { - zombienet.build_specs()?; + zombienet.build_container_chain_specs("tanssi-node")?; } // Finally spawn network and wait for signal to terminate diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 999ff207..819218ca 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -121,60 +121,45 @@ pub fn generate_raw_chain_spec( Ok(raw_chain_spec) } -/// Generates a raw chain specification file for a parachain. +/// Generates a raw chain specification file for container chain. /// /// # Arguments /// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. -/// * `plain_chain_spec` - Location of the plain chain specification file. -/// * `chain_spec_file_name` - The name of the chain specification file to be generated. -pub fn generate_raw_chain_spec_tanssi( +/// * `chain` - An optional chain to be used. This should be provided if generating a spec for a parachain. +/// * `container_chains` - An optional vector of container chain specification files. These should be provided if generating a spec for a parachain. +/// * `id` - The parachain id for which the chain specification is being generated. +/// * `invulnerables` - An optional vector of invulnerable nodes. These should be provided if generating a spec for a parachain. +/// * `raw_chain_spec` - The path where the generated raw chain specification file will be generated. +pub fn generate_raw_chain_spec_container_chain( binary_path: &Path, chain: Option<&str>, + container_chains: Option>, + id: &str, + invulnerables: Option>, raw_chain_spec: &Path, - container_chains: Vec<&str>, - invulnerables: Vec<&str>, - parachain_id: &str, ) -> Result { check_command_exists(&binary_path, "build-spec")?; - let mut args = vec!["build-spec", "--parachain-id", parachain_id]; - if let Some(chain) = chain { - args.push("--chain"); - args.push(chain); - } - for container_chain in container_chains { - args.push("--add-container-chain"); - args.push(container_chain); + let mut args = vec!["build-spec", "--parachain-id", id]; + // Parachain + if let (Some(chain), Some(invulnerables), Some(container_chains)) = + (chain, invulnerables, container_chains) + { + args.extend(&["--chain", chain]); + for container_chain in container_chains { + args.extend(&["--add-container-chain", container_chain]); + } + for invulnerable in invulnerables { + args.extend(&["--invulnerable", invulnerable]); + } } - for invulnerable in invulnerables { - args.push("--invulnerable"); - args.push(invulnerable); + // Container Chain + else { + args.extend(["--disable-default-bootnode", "--raw"]); } cmd(binary_path, args).stdout_path(raw_chain_spec).stderr_null().run()?; Ok(raw_chain_spec.to_path_buf()) } -/// Generates a raw chain specification file for a parachain. -/// -/// # Arguments -/// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. -/// * `plain_chain_spec` - Location of the plain chain specification file. -/// * `chain_spec_file_name` - The name of the chain specification file to be generated. -pub fn generate_raw_chain_spec_container_chain( - binary_path: &Path, - raw_chain_spec: &Path, - parachain_id: &str, -) -> Result { - check_command_exists(&binary_path, "build-spec")?; - cmd( - binary_path, - vec!["build-spec", "--disable-default-bootnode", "--parachain-id", parachain_id, "--raw"], - ) - .stderr_null() - .stdout_path(&raw_chain_spec) - .run()?; - Ok(raw_chain_spec.to_path_buf()) -} - /// Export the WebAssembly runtime for the parachain. /// /// # Arguments diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index c3376e7a..016383ac 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -9,7 +9,7 @@ pub enum Error { Aborted, #[error("Anyhow error: {0}")] AnyhowError(#[from] anyhow::Error), - #[error("Missing chain with id: {0}")] + #[error("Missing chain: {0}")] ChainNotFound(String), #[error("{0}")] CommonError(#[from] pop_common::Error), diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 60df6225..3b6e49be 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-3.0 -use crate::{ - build::{generate_raw_chain_spec_container_chain, generate_raw_chain_spec_tanssi}, - errors::Error, -}; +use crate::{build::generate_raw_chain_spec_container_chain, errors::Error}; use glob::glob; use indexmap::IndexMap; pub use pop_common::{ @@ -103,20 +100,29 @@ impl Zombienet { .filter_map(|b| b) } - /// Build the needed specs before spawn the network. - pub fn build_specs(&mut self) -> Result<(), Error> { - // This order is important, first container chains that can be more than one, then tanssi-node + /// Build the needed specs before spawn the network containing the container chains. + pub fn build_container_chain_specs(&mut self, parachain_name: &str) -> Result<(), Error> { + let parachain_id = self + .parachains + .iter() + .find(|(_, parachain)| parachain.binary.name() == parachain_name) + .ok_or(Error::ChainNotFound(format!("with name: {parachain_name}")))? + .0; + // This order is important, first container chains that can be more than one, then the parachain let mut container_chains: Vec = Vec::new(); - self.parachains.iter().filter(|(&id, _)| id > 1000).try_for_each( + self.parachains.iter().filter(|(&id, _)| &id > parachain_id).try_for_each( |(id, parachain)| -> Result<(), Error> { container_chains.push( generate_raw_chain_spec_container_chain( parachain.binary.path().as_path(), + None, + None, + &id.to_string(), + None, parachain .chain_spec_path .as_deref() .unwrap_or(Path::new("./chain-spec.json")), - &id.to_string(), )? .display() .to_string(), @@ -124,19 +130,24 @@ impl Zombienet { Ok(()) }, )?; - let tanssi_node = self.parachains.get(&1000).ok_or(Error::ChainNotFound("1000".into()))?; - generate_raw_chain_spec_tanssi( + let tanssi_node = self + .parachains + .get(parachain_id) + .ok_or(Error::ChainNotFound(format!("with id: {parachain_id}")))?; + generate_raw_chain_spec_container_chain( tanssi_node.binary.path().as_path(), tanssi_node.chain.as_deref(), + Some(container_chains.iter().map(|s| s.as_str()).collect()), + ¶chain_id.to_string(), + Some( + tanssi_node + .collators_names + .iter() + .filter(|s| s.starts_with("Collator")) + .map(|s| s.as_str()) + .collect(), + ), tanssi_node.chain_spec_path.as_deref().unwrap_or(Path::new("./chain-spec.json")), - container_chains.iter().map(|s| s.as_str()).collect(), - tanssi_node - .collators_names - .iter() - .filter(|s| s.starts_with("Collator")) - .map(|s| s.as_str()) - .collect(), - &1000.to_string(), )?; Ok(()) } From 71b83a6b7f28fc1008b5d59f9b25c7f11d84d2ec Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 22:39:35 +0200 Subject: [PATCH 04/19] test: fix unit tests --- crates/pop-parachains/src/up/mod.rs | 51 ++++++++++++++++++++-- crates/pop-parachains/src/up/parachains.rs | 18 ++++++-- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 3b6e49be..03ed786c 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -1562,6 +1562,7 @@ validator = true fs::{create_dir_all, File}, io::{Read, Write}, path::PathBuf, + vec, }; use tempfile::{tempdir, Builder}; @@ -1712,6 +1713,8 @@ command = "./target/release/parachain-template-node" }, chain: None, chain_spec_generator: None, + chain_spec_path: None, + collators_names: vec!["asset-hub".to_string()], }, ), ( @@ -1725,6 +1728,8 @@ command = "./target/release/parachain-template-node" }, chain: None, chain_spec_generator: None, + chain_spec_path: None, + collators_names: vec!["pop".to_string()], }, ), ( @@ -1738,6 +1743,8 @@ command = "./target/release/parachain-template-node" }, chain: None, chain_spec_generator: None, + chain_spec_path: None, + collators_names: vec!["collator".to_string()], }, ), ] @@ -1867,6 +1874,8 @@ command = "polkadot-parachain" path: system_chain_spec_generator.to_path_buf(), manifest: None, }), + chain_spec_path: None, + collators_names: vec!["asset-hub".to_string()], }, )] .into(), @@ -1945,12 +1954,20 @@ node_spawn_timeout = 300 let name = "parachain-template-node"; let command = PathBuf::from("./target/release").join(&name); assert_eq!( - Parachain::from_local(2000, command.clone(), Some("dev"))?, + Parachain::from_local( + 2000, + command.clone(), + Some("dev"), + Some("/path"), + vec![name] + )?, Parachain { id: 2000, binary: Binary::Local { name: name.to_string(), path: command, manifest: None }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec![name.to_string()], } ); Ok(()) @@ -1961,7 +1978,13 @@ node_spawn_timeout = 300 let name = "pop-parachains"; let command = PathBuf::from("./target/release").join(&name); assert_eq!( - Parachain::from_local(2000, command.clone(), Some("dev"))?, + Parachain::from_local( + 2000, + command.clone(), + Some("dev"), + Some("/path"), + vec![name] + )?, Parachain { id: 2000, binary: Binary::Local { @@ -1971,6 +1994,8 @@ node_spawn_timeout = 300 }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec![name.to_string()], } ); Ok(()) @@ -1981,7 +2006,14 @@ node_spawn_timeout = 300 let repo = Repository::parse("https://git.com/r0gue-io/pop-node#v1.0")?; let cache = tempdir()?; assert_eq!( - Parachain::from_repository(2000, &repo, Some("dev"), cache.path())?, + Parachain::from_repository( + 2000, + &repo, + Some("dev"), + cache.path(), + Some("/path"), + vec!["pop-node"] + )?, Parachain { id: 2000, binary: Binary::Source { @@ -1997,6 +2029,8 @@ node_spawn_timeout = 300 }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec!["pop-node".to_string()], } ); Ok(()) @@ -2007,7 +2041,14 @@ node_spawn_timeout = 300 let repo = Repository::parse("https://github.com/r0gue-io/pop-node#v1.0")?; let cache = tempdir()?; assert_eq!( - Parachain::from_repository(2000, &repo, Some("dev"), cache.path())?, + Parachain::from_repository( + 2000, + &repo, + Some("dev"), + cache.path(), + Some("/path"), + vec!["pop-node"] + )?, Parachain { id: 2000, binary: Binary::Source { @@ -2024,6 +2065,8 @@ node_spawn_timeout = 300 }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec!["pop-node".to_string()], }, ); Ok(()) diff --git a/crates/pop-parachains/src/up/parachains.rs b/crates/pop-parachains/src/up/parachains.rs index e63c3782..5cbc8230 100644 --- a/crates/pop-parachains/src/up/parachains.rs +++ b/crates/pop-parachains/src/up/parachains.rs @@ -274,9 +274,17 @@ mod tests { let para_id = 2000; let temp_dir = tempdir()?; - let parachain = from(para_id, expected.binary(), Some(version), None, temp_dir.path()) - .await? - .unwrap(); + let parachain = from( + para_id, + expected.binary(), + Some(version), + None, + temp_dir.path(), + Some("/path"), + vec!["pop-node"], + ) + .await? + .unwrap(); assert_eq!(para_id, parachain.id); assert!(matches!(parachain.binary, Binary::Source { name, source, cache } if name == expected.binary() && source == Source::GitHub(ReleaseArchive { @@ -294,7 +302,9 @@ mod tests { #[tokio::test] async fn from_handles_unsupported_command() -> anyhow::Result<()> { - assert!(from(2000, "none", None, None, &PathBuf::default()).await?.is_none()); + assert!(from(2000, "none", None, None, &PathBuf::default(), None, vec!["pop-node"]) + .await? + .is_none()); Ok(()) } } From 4084e39894bed061f0a16eb0c82c354d22ee9cfb Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 22:41:29 +0200 Subject: [PATCH 05/19] refactor: network.templ for container chains --- .../templates/container/network.templ | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/pop-parachains/templates/container/network.templ b/crates/pop-parachains/templates/container/network.templ index e495c281..fc69b942 100644 --- a/crates/pop-parachains/templates/container/network.templ +++ b/crates/pop-parachains/templates/container/network.templ @@ -20,26 +20,31 @@ validator = true [[parachains]] id = 1000 chain = "dancebox-local" +chain_spec_path = "tanssi-1000.json" default_command = "tanssi-node" [[parachains.collators]] -name = "collator-01" +name = "FullNode-1000" [[parachains.collators]] -name = "collator-02" +name = "Collator1000-01" [[parachains.collators]] -name = "collator-03" +name = "Collator1000-02" [[parachains.collators]] -name = "collator-04" +name = "Collator2000-01" + +[[parachains.collators]] +name = "Collator2000-02" [[parachains]] id = 2000 +chain_spec_path = "template-container-2000.json" default_command = "./target/release/^^node^^" [[parachains.collators]] -name = "full-node-01" +name = "FullNode-2000" [[parachains.collators]] -name = "full-node-02" +name = "FullNode-2000-2" From 3947aab66b92228ea744d7c12b5fb241eb893189 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Wed, 4 Sep 2024 10:56:59 +0200 Subject: [PATCH 06/19] test: unit tests --- crates/pop-parachains/src/build.rs | 88 +++++++++++++++++++- crates/pop-parachains/src/errors.rs | 2 +- crates/pop-parachains/src/new_parachain.rs | 2 +- crates/pop-parachains/src/up/mod.rs | 94 +++++++++++++++++++++- 4 files changed, 179 insertions(+), 7 deletions(-) diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 819218ca..91412d20 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -356,8 +356,9 @@ impl ChainSpec { mod tests { use super::*; use crate::{ - new_parachain::instantiate_standard_template, templates::Parachain, Config, Error, - Zombienet, + new_parachain::{instantiate_standard_template, instantiate_tanssi_template}, + templates::Parachain, + Config, Error, Zombienet, }; use anyhow::Result; use pop_common::manifest::Dependency; @@ -531,6 +532,89 @@ default_command = "pop-node" Ok(()) } + #[tokio::test] + async fn generate_raw_chain_spec_container_chain_works() -> Result<()> { + let temp_dir = tempdir().expect("Failed to create temp dir"); + instantiate_tanssi_template(&Parachain::TanssiSimple, temp_dir.path(), None)?; + let config = Builder::new().suffix(".toml").tempfile()?; + // Get Zombienet config and fet binary for generate specs + writeln!( + config.as_file(), + "{}", + format!( + r#" +[relaychain] +chain = "paseo-local" + +[[parachains]] +id = 1000 +chain_spec_path = "{}" +chain = "dancebox-local" +default_command = "tanssi-node" + +[[parachains.collators]] +name = "FullNode-1000" + +[[parachains.collators]] +name = "Collator1000-01" + +[[parachains.collators]] +name = "Collator1000-02" + +[[parachains.collators]] +name = "Collator2000-01" + +[[parachains.collators]] +name = "Collator2000-02" +"#, + &temp_dir.path().join("tanssi-1000.json").display().to_string() + ) + )?; + let mut zombienet = Zombienet::new( + &temp_dir.path(), + config.path().to_str().unwrap(), + None, + None, + None, + None, + None, + ) + .await?; + // Fetch the tanssi-node binary + let mut binary_name: String = "".to_string(); + for binary in zombienet.binaries().filter(|b| !b.exists() && b.name() == "tanssi-node") { + binary_name = format!("{}-{}", binary.name(), binary.latest().unwrap()); + binary.source(true, &(), true).await?; + } + + let raw_chain_spec_container_chain = generate_raw_chain_spec_container_chain( + &temp_dir.path().join(binary_name.clone()), + None, + None, + &"2000", + None, + &temp_dir.path().join("raw-container-chain-chainspec.json"), + )?; + assert!(raw_chain_spec_container_chain.exists()); + + let raw_chain_spec = generate_raw_chain_spec_container_chain( + &temp_dir.path().join(binary_name), + Some("dancebox-local"), + Some(vec![&temp_dir + .path() + .join("raw-container-chain-chainspec.json") + .display() + .to_string()]), + &"1000", + Some(vec!["Collator1000-01", "Collator1000-02", "Collator2000-01", "Collator2000-02"]), + &temp_dir.path().join("raw-parachain-chainspec.json"), + )?; + assert!(raw_chain_spec.exists()); + let content = fs::read_to_string(raw_chain_spec.clone()).expect("Could not read file"); + assert!(content.contains("\"para_id\": 2000")); + Ok(()) + } + #[test] fn raw_chain_spec_fails_wrong_chain_spec() -> Result<()> { assert!(matches!( diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index 016383ac..0e3fee1c 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -9,7 +9,7 @@ pub enum Error { Aborted, #[error("Anyhow error: {0}")] AnyhowError(#[from] anyhow::Error), - #[error("Missing chain: {0}")] + #[error("Missing chain {0}")] ChainNotFound(String), #[error("{0}")] CommonError(#[from] pop_common::Error), diff --git a/crates/pop-parachains/src/new_parachain.rs b/crates/pop-parachains/src/new_parachain.rs index 25a71353..3fbb289b 100644 --- a/crates/pop-parachains/src/new_parachain.rs +++ b/crates/pop-parachains/src/new_parachain.rs @@ -94,7 +94,7 @@ pub fn instantiate_standard_template( /// * `template` - template to generate the container-chain from. /// * `target` - location where the parachain will be created. /// * `tag_version` - version to use (`None` to use latest). -fn instantiate_tanssi_template( +pub fn instantiate_tanssi_template( template: &ContainerChain, target: &Path, tag_version: Option, diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 03ed786c..cd44afa5 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -101,7 +101,7 @@ impl Zombienet { } /// Build the needed specs before spawn the network containing the container chains. - pub fn build_container_chain_specs(&mut self, parachain_name: &str) -> Result<(), Error> { + pub fn build_container_chain_specs(&mut self, parachain_name: &str) -> Result { let parachain_id = self .parachains .iter() @@ -134,7 +134,7 @@ impl Zombienet { .parachains .get(parachain_id) .ok_or(Error::ChainNotFound(format!("with id: {parachain_id}")))?; - generate_raw_chain_spec_container_chain( + let raw_spec_path = generate_raw_chain_spec_container_chain( tanssi_node.binary.path().as_path(), tanssi_node.chain.as_deref(), Some(container_chains.iter().map(|s| s.as_str()).collect()), @@ -149,7 +149,7 @@ impl Zombienet { ), tanssi_node.chain_spec_path.as_deref().unwrap_or(Path::new("./chain-spec.json")), )?; - Ok(()) + Ok(raw_spec_path) } /// Determine parachain configuration based on specified version and network configuration. @@ -768,11 +768,16 @@ fn resolve_manifest(package: &str, path: &Path) -> Result, Error #[cfg(test)] mod tests { use super::*; + use crate::{ + new_parachain::instantiate_tanssi_template, templates::Parachain as Template, Error, + }; use anyhow::Result; use std::{env::current_dir, fs::File, io::Write}; use tempfile::tempdir; mod zombienet { + use std::fs; + use super::*; use pop_common::Status; @@ -1416,6 +1421,89 @@ chain = "asset-hub-paseo-local" Ok(()) } + #[tokio::test] + async fn build_container_chain_specs_works() -> Result<()> { + let temp_dir = tempdir().expect("Failed to create temp dir"); + instantiate_tanssi_template(&Template::TanssiSimple, temp_dir.path(), None)?; + let config = Builder::new().suffix(".toml").tempfile()?; + // Get Zombienet config and fet binary for generate specs + writeln!( + config.as_file(), + "{}", + format!( + r#" +[relaychain] +chain = "paseo-local" + +[[parachains]] +id = 1000 +chain_spec_path = "{}" +chain = "dancebox-local" +default_command = "tanssi-node" + +[[parachains.collators]] +name = "FullNode-1000" + +[[parachains.collators]] +name = "Collator1000-01" + +[[parachains.collators]] +name = "Collator1000-02" + +[[parachains.collators]] +name = "Collator2000-01" + +[[parachains.collators]] +name = "Collator2000-02" +"#, + &temp_dir.path().join("tanssi-1000.json").display().to_string() + ) + )?; + let mut zombienet = Zombienet::new( + &temp_dir.path(), + config.path().to_str().unwrap(), + None, + None, + None, + None, + None, + ) + .await?; + // Fetch the tanssi-node binary + for binary in zombienet.binaries().filter(|b| !b.exists() && b.name() == "tanssi-node") + { + binary.source(true, &(), true).await?; + } + let raw_chain_spec = zombienet.build_container_chain_specs("tanssi-node")?; + assert!(raw_chain_spec.exists()); + let content = fs::read_to_string(raw_chain_spec.clone()).expect("Could not read file"); + assert!(content.contains("\"para_id\": 1000")); + assert!(content.contains("\"id\": \"dancebox_local\"")); + Ok(()) + } + + #[tokio::test] + async fn build_container_chain_specs_fails_chain_not_found() -> Result<()> { + let temp_dir = tempdir().expect("Failed to create temp dir"); + instantiate_tanssi_template(&Template::TanssiSimple, temp_dir.path(), None)?; + let mut zombienet = Zombienet::new( + &temp_dir.path(), + &temp_dir.path().join("network.toml").display().to_string(), + None, + None, + None, + None, + None, + ) + .await?; + assert!(matches!( + zombienet.build_container_chain_specs("bad-name"), + Err(Error::ChainNotFound(error)) + if error == "with name: bad-name" + )); + Ok(()) + } + #[tokio::test] async fn spawn_ensures_relay_chain_binary_exists() -> Result<()> { let temp_dir = tempdir()?; From e660acb4e1b4855db4864b87fcef5f980e4af2dd Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Wed, 4 Sep 2024 16:42:55 +0200 Subject: [PATCH 07/19] fix: clone_and_degit to fetch from latest tag instead of master --- crates/pop-common/src/git.rs | 42 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/crates/pop-common/src/git.rs b/crates/pop-common/src/git.rs index 2e6b205a..e60aca7a 100644 --- a/crates/pop-common/src/git.rs +++ b/crates/pop-common/src/git.rs @@ -66,32 +66,38 @@ impl Git { target, )?, }; - + // Checkout the specific tag if provided if let Some(tag_version) = tag_version { - let (object, reference) = repo.revparse_ext(&tag_version).expect("Object not found"); - repo.checkout_tree(&object, None).expect("Failed to checkout"); - match reference { - // gref is an actual reference like branches or tags - Some(gref) => repo.set_head(gref.name().unwrap()), - // this is a commit, not a reference - None => repo.set_head_detached(object.id()), - } - .expect("Failed to set HEAD"); - - let git_dir = repo.path(); - fs::remove_dir_all(&git_dir)?; - return Ok(Some(tag_version)); + return Self::git_checkout(&repo, tag_version); } - - // fetch tags from remote + // If not to the latest release let release = Self::fetch_latest_tag(&repo); - + if let Some(tag_version) = release { + return Self::git_checkout(&repo, tag_version); + } + // If no tags, take it from main branch let git_dir = repo.path(); fs::remove_dir_all(&git_dir)?; - // Or by default the last one Ok(release) } + // Checkout to a specific tag or commit from a Git repository. + fn git_checkout(repo: &GitRepository, tag_version: String) -> Result> { + let (object, reference) = repo.revparse_ext(&tag_version).expect("Object not found"); + repo.checkout_tree(&object, None).expect("Failed to checkout"); + match reference { + // gref is an actual reference like branches or tags + Some(gref) => repo.set_head(gref.name().unwrap()), + // this is a commit, not a reference + None => repo.set_head_detached(object.id()), + } + .expect("Failed to set HEAD"); + + let git_dir = repo.path(); + fs::remove_dir_all(&git_dir)?; + Ok(Some(tag_version)) + } + /// For users that have ssh configuration for cloning repositories. fn ssh_clone_and_degit(url: Url, target: &Path) -> Result { let ssh_url = GitHub::convert_to_ssh_url(&url); From 64d657e12ec25d81a048589347e1822dba449b88 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Wed, 4 Sep 2024 20:13:15 +0200 Subject: [PATCH 08/19] refactor clone°it --- crates/pop-common/src/git.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/pop-common/src/git.rs b/crates/pop-common/src/git.rs index e60aca7a..8f7c933a 100644 --- a/crates/pop-common/src/git.rs +++ b/crates/pop-common/src/git.rs @@ -66,19 +66,21 @@ impl Git { target, )?, }; - // Checkout the specific tag if provided - if let Some(tag_version) = tag_version { - return Self::git_checkout(&repo, tag_version); - } - // If not to the latest release - let release = Self::fetch_latest_tag(&repo); - if let Some(tag_version) = release { - return Self::git_checkout(&repo, tag_version); + + match tag_version { + Some(tag) => Self::git_checkout(&repo, tag), + None => { + // Fetch latest release. + let release = Self::fetch_latest_tag(&repo); + if let Some(r) = release { + return Self::git_checkout(&repo, r); + } + // If there are no releases, use main. + let git_dir = repo.path(); + fs::remove_dir_all(&git_dir)?; + Ok(release) + }, } - // If no tags, take it from main branch - let git_dir = repo.path(); - fs::remove_dir_all(&git_dir)?; - Ok(release) } // Checkout to a specific tag or commit from a Git repository. From f33c00b31fccc56bca19ab19c28cd31516d7f76b Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:58:53 +0200 Subject: [PATCH 09/19] source tanssi-node --- crates/pop-parachains/src/up/parachains.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/pop-parachains/src/up/parachains.rs b/crates/pop-parachains/src/up/parachains.rs index 3bea2cba..bae8adec 100644 --- a/crates/pop-parachains/src/up/parachains.rs +++ b/crates/pop-parachains/src/up/parachains.rs @@ -32,6 +32,13 @@ pub(super) enum Parachain { Fallback = "v0.1.0-alpha2" ))] Pop, + /// Swift and effortless deployment of application-specific blockchains. + #[strum(props( + Repository = "https://github.com/r0gue-io/tanssi", + Binary = "tanssi-node", + Fallback = "v0.8.1" + ))] + Tanssi, } impl TryInto for Parachain { @@ -42,7 +49,7 @@ impl TryInto for Parachain { /// * `latest` - If applicable, some specifier used to determine the latest source. fn try_into(&self, tag: Option, latest: Option) -> Result { Ok(match self { - Parachain::System | Parachain::Pop => { + Parachain::System | Parachain::Pop | Parachain::Tanssi => { // Source from GitHub release asset let repo = GitHub::parse(self.repository())?; Source::GitHub(ReleaseArchive { @@ -135,6 +142,7 @@ pub(super) async fn from( source: TryInto::try_into(para, tag, latest)?, cache: cache.to_path_buf(), }; + println!("{:?}", &chain.unwrap()); return Ok(Some(super::Parachain { id, binary, From 63e242d8eab0dee9e70f9167b6b42b48a0e5cbb2 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 21:07:21 +0200 Subject: [PATCH 10/19] feat: build specs for tanssi pop up --- crates/pop-cli/src/commands/up/parachain.rs | 3 + crates/pop-parachains/src/build.rs | 54 +++++++++ crates/pop-parachains/src/errors.rs | 2 + crates/pop-parachains/src/up/mod.rs | 117 +++++++++++++++++++- crates/pop-parachains/src/up/parachains.rs | 9 +- 5 files changed, 177 insertions(+), 8 deletions(-) diff --git a/crates/pop-cli/src/commands/up/parachain.rs b/crates/pop-cli/src/commands/up/parachain.rs index e68d20a1..55b86f83 100644 --- a/crates/pop-cli/src/commands/up/parachain.rs +++ b/crates/pop-cli/src/commands/up/parachain.rs @@ -89,6 +89,9 @@ impl ZombienetCommand { if Self::source_binaries(&mut zombienet, &cache, self.verbose, self.skip_confirm).await? { return Ok(()); } + if zombienet.binaries().any(|b| b.name() == "tanssi-node") { + zombienet.build_specs()?; + } // Finally spawn network and wait for signal to terminate let spinner = cliclack::spinner(); diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 38d760c5..44b7f744 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -121,6 +121,60 @@ pub fn generate_raw_chain_spec( Ok(raw_chain_spec) } +/// Generates a raw chain specification file for a parachain. +/// +/// # Arguments +/// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. +/// * `plain_chain_spec` - Location of the plain chain specification file. +/// * `chain_spec_file_name` - The name of the chain specification file to be generated. +pub fn generate_raw_chain_spec_tanssi( + binary_path: &Path, + chain: Option<&str>, + raw_chain_spec: &Path, + container_chains: Vec<&str>, + invulnerables: Vec<&str>, + parachain_id: &str, +) -> Result { + check_command_exists(&binary_path, "build-spec")?; + let mut args = vec!["build-spec", "--parachain-id", parachain_id]; + if let Some(chain) = chain { + args.push("--chain"); + args.push(chain); + } + for container_chain in container_chains { + args.push("--add-container-chain"); + args.push(container_chain); + } + for invulnerable in invulnerables { + args.push("--invulnerable"); + args.push(invulnerable); + } + cmd(binary_path, args).stdout_path(raw_chain_spec).stderr_null().run()?; + Ok(raw_chain_spec.to_path_buf()) +} + +/// Generates a raw chain specification file for a parachain. +/// +/// # Arguments +/// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. +/// * `plain_chain_spec` - Location of the plain chain specification file. +/// * `chain_spec_file_name` - The name of the chain specification file to be generated. +pub fn generate_raw_chain_spec_container_chain( + binary_path: &Path, + raw_chain_spec: &Path, + parachain_id: &str, +) -> Result { + check_command_exists(&binary_path, "build-spec")?; + cmd( + binary_path, + vec!["build-spec", "--disable-default-bootnode", "--parachain-id", parachain_id, "--raw"], + ) + .stderr_null() + .stdout_path(&raw_chain_spec) + .run()?; + Ok(raw_chain_spec.to_path_buf()) +} + /// Export the WebAssembly runtime for the parachain. /// /// # Arguments diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index 58b59c58..c3376e7a 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -9,6 +9,8 @@ pub enum Error { Aborted, #[error("Anyhow error: {0}")] AnyhowError(#[from] anyhow::Error), + #[error("Missing chain with id: {0}")] + ChainNotFound(String), #[error("{0}")] CommonError(#[from] pop_common::Error), #[error("Configuration error: {0}")] diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 192da7ff..61bee610 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-3.0 -use crate::errors::Error; +use crate::{ + build::{generate_raw_chain_spec_container_chain, generate_raw_chain_spec_tanssi}, + errors::Error, +}; use glob::glob; use indexmap::IndexMap; pub use pop_common::{ @@ -100,6 +103,44 @@ impl Zombienet { .filter_map(|b| b) } + /// Build the needed specs before spawn the network. + pub fn build_specs(&mut self) -> Result<(), Error> { + // This order is important, first container chains that can be more than one, then tanssi-node + let mut container_chains: Vec = Vec::new(); + self.parachains.iter().filter(|(&id, _)| id > 1000).try_for_each( + |(id, parachain)| -> Result<(), Error> { + container_chains.push( + generate_raw_chain_spec_container_chain( + parachain.binary.path().as_path(), + parachain + .chain_spec_path + .as_deref() + .unwrap_or(Path::new("./chain-spec.json")), + &id.to_string(), + )? + .display() + .to_string(), + ); + Ok(()) + }, + )?; + let tanssi_node = self.parachains.get(&1000).ok_or(Error::ChainNotFound("1000".into()))?; + generate_raw_chain_spec_tanssi( + tanssi_node.binary.path().as_path(), + tanssi_node.chain.as_deref(), + tanssi_node.chain_spec_path.as_deref().unwrap_or(Path::new("./chain-spec.json")), + container_chains.iter().map(|s| s.as_str()).collect(), + tanssi_node + .collators_names + .iter() + .filter(|s| s.starts_with("Collator")) + .map(|s| s.as_str()) + .collect(), + &1000.to_string(), + )?; + Ok(()) + } + /// Determine parachain configuration based on specified version and network configuration. /// /// # Arguments @@ -132,7 +173,7 @@ impl Zombienet { as u32; let chain = table.get("chain").and_then(|i| i.as_str()); - + let chain_spec_path = table.get("chain_spec_path").and_then(|i| i.as_str()); let command = NetworkConfiguration::default_command(table) .cloned() .or_else(|| { @@ -175,6 +216,18 @@ impl Zombienet { continue; } + let mut collators_names: Vec<&str> = Vec::new(); + if let Some(collators) = table.get("collators").and_then(|p| p.as_array_of_tables()) { + for collator in collators.iter() { + if let Some(name) = + NetworkConfiguration::name(collator).and_then(|i| i.as_str()) + { + //let n = Some(Item::Value(Value::String(Formatted::new(name.into())))); + collators_names.push(name); + } + } + } + // Check if known parachain let version = parachains.as_ref().and_then(|r| { r.iter() @@ -182,7 +235,17 @@ impl Zombienet { .nth(0) .map(|v| v.as_str()) }); - if let Some(parachain) = parachains::from(id, &command, version, chain, cache).await? { + if let Some(parachain) = parachains::from( + id, + &command, + version, + chain, + cache, + chain_spec_path, + collators_names.clone(), + ) + .await? + { paras.insert(id, parachain); continue; } @@ -190,14 +253,33 @@ impl Zombienet { // Check if parachain binary source specified as an argument if let Some(parachains) = parachains.as_ref() { for repo in parachains.iter().filter(|r| command == r.package) { - paras.insert(id, Parachain::from_repository(id, repo, chain, cache)?); + paras.insert( + id, + Parachain::from_repository( + id, + repo, + chain, + cache, + chain_spec_path, + collators_names, + )?, + ); continue 'outer; } } // Check if command references a local binary if ["./", "../", "/"].iter().any(|p| command.starts_with(p)) { - paras.insert(id, Parachain::from_local(id, command.into(), chain)?); + paras.insert( + id, + Parachain::from_local( + id, + command.into(), + chain, + chain_spec_path, + collators_names, + )?, + ); continue; } @@ -368,6 +450,11 @@ impl NetworkConfiguration { config.get("default_command") } + /// Returns the `name` configuration. + fn name(config: &Table) -> Option<&Item> { + config.get("name") + } + /// Returns the `nodes` configuration. fn nodes(relay_chain: &Table) -> Option<&ArrayOfTables> { relay_chain.get("nodes").and_then(|i| i.as_array_of_tables()) @@ -506,6 +593,10 @@ struct Parachain { chain: Option, /// If applicable, the binary used to generate a chain specification. chain_spec_generator: Option, + /// If applicable, the path to the chain specification. + chain_spec_path: Option, + /// List of collators names. + collators_names: Vec, } impl Parachain { @@ -515,7 +606,13 @@ impl Parachain { /// * `id` - The parachain identifier on the local network. /// * `path` - The path to the local binary. /// * `chain` - The chain specified. - fn from_local(id: u32, path: PathBuf, chain: Option<&str>) -> Result { + fn from_local( + id: u32, + path: PathBuf, + chain: Option<&str>, + chain_spec_path: Option<&str>, + collators_names: Vec<&str>, + ) -> Result { let name = path .file_name() .and_then(|f| f.to_str()) @@ -528,6 +625,8 @@ impl Parachain { binary: Binary::Local { name, path, manifest }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), }) } @@ -544,6 +643,8 @@ impl Parachain { repo: &Repository, chain: Option<&str>, cache: &Path, + chain_spec_path: Option<&str>, + collators_names: Vec<&str>, ) -> Result { // Check for GitHub repository to be able to download source as an archive if repo.url.host_str().is_some_and(|h| h.to_lowercase() == "github.com") { @@ -565,6 +666,8 @@ impl Parachain { }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), }) } else { Ok(Parachain { @@ -582,6 +685,8 @@ impl Parachain { }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), }) } } diff --git a/crates/pop-parachains/src/up/parachains.rs b/crates/pop-parachains/src/up/parachains.rs index bae8adec..e63c3782 100644 --- a/crates/pop-parachains/src/up/parachains.rs +++ b/crates/pop-parachains/src/up/parachains.rs @@ -9,7 +9,7 @@ use pop_common::{ }, target, Error, GitHub, }; -use std::path::Path; +use std::path::{Path, PathBuf}; use strum::VariantArray as _; use strum_macros::{EnumProperty, VariantArray}; @@ -111,6 +111,8 @@ pub(super) async fn system( binary, chain: chain.map(|c| c.to_string()), chain_spec_generator, + chain_spec_path: None, + collators_names: Vec::new(), })); } @@ -128,6 +130,8 @@ pub(super) async fn from( version: Option<&str>, chain: Option<&str>, cache: &Path, + chain_spec_path: Option<&str>, + collators_names: Vec<&str>, ) -> Result, Error> { for para in Parachain::VARIANTS.iter().filter(|p| p.binary() == command) { let releases = para.releases().await?; @@ -142,12 +146,13 @@ pub(super) async fn from( source: TryInto::try_into(para, tag, latest)?, cache: cache.to_path_buf(), }; - println!("{:?}", &chain.unwrap()); return Ok(Some(super::Parachain { id, binary, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_path: chain_spec_path.map(PathBuf::from), + collators_names: collators_names.iter().map(|&s| s.to_string()).collect(), })); } Ok(None) From a8427c7c5057efb268594ef4b60ef3d1bb594ff7 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 22:29:46 +0200 Subject: [PATCH 11/19] refactor: build specs container chains --- crates/pop-cli/src/commands/up/parachain.rs | 4 +- crates/pop-parachains/src/build.rs | 65 ++++++++------------- crates/pop-parachains/src/errors.rs | 2 +- crates/pop-parachains/src/up/mod.rs | 49 ++++++++++------ 4 files changed, 59 insertions(+), 61 deletions(-) diff --git a/crates/pop-cli/src/commands/up/parachain.rs b/crates/pop-cli/src/commands/up/parachain.rs index 55b86f83..a80efb61 100644 --- a/crates/pop-cli/src/commands/up/parachain.rs +++ b/crates/pop-cli/src/commands/up/parachain.rs @@ -89,8 +89,10 @@ impl ZombienetCommand { if Self::source_binaries(&mut zombienet, &cache, self.verbose, self.skip_confirm).await? { return Ok(()); } + // For container chains, build the specs before spawn the network + // TODO: Is there a better way to identifiy it than to check if there is a hardcoded tanssi-node if zombienet.binaries().any(|b| b.name() == "tanssi-node") { - zombienet.build_specs()?; + zombienet.build_container_chain_specs("tanssi-node")?; } // Finally spawn network and wait for signal to terminate diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 44b7f744..4d61f829 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -121,60 +121,45 @@ pub fn generate_raw_chain_spec( Ok(raw_chain_spec) } -/// Generates a raw chain specification file for a parachain. +/// Generates a raw chain specification file for container chain. /// /// # Arguments /// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. -/// * `plain_chain_spec` - Location of the plain chain specification file. -/// * `chain_spec_file_name` - The name of the chain specification file to be generated. -pub fn generate_raw_chain_spec_tanssi( +/// * `chain` - An optional chain to be used. This should be provided if generating a spec for a parachain. +/// * `container_chains` - An optional vector of container chain specification files. These should be provided if generating a spec for a parachain. +/// * `id` - The parachain id for which the chain specification is being generated. +/// * `invulnerables` - An optional vector of invulnerable nodes. These should be provided if generating a spec for a parachain. +/// * `raw_chain_spec` - The path where the generated raw chain specification file will be generated. +pub fn generate_raw_chain_spec_container_chain( binary_path: &Path, chain: Option<&str>, + container_chains: Option>, + id: &str, + invulnerables: Option>, raw_chain_spec: &Path, - container_chains: Vec<&str>, - invulnerables: Vec<&str>, - parachain_id: &str, ) -> Result { check_command_exists(&binary_path, "build-spec")?; - let mut args = vec!["build-spec", "--parachain-id", parachain_id]; - if let Some(chain) = chain { - args.push("--chain"); - args.push(chain); - } - for container_chain in container_chains { - args.push("--add-container-chain"); - args.push(container_chain); + let mut args = vec!["build-spec", "--parachain-id", id]; + // Parachain + if let (Some(chain), Some(invulnerables), Some(container_chains)) = + (chain, invulnerables, container_chains) + { + args.extend(&["--chain", chain]); + for container_chain in container_chains { + args.extend(&["--add-container-chain", container_chain]); + } + for invulnerable in invulnerables { + args.extend(&["--invulnerable", invulnerable]); + } } - for invulnerable in invulnerables { - args.push("--invulnerable"); - args.push(invulnerable); + // Container Chain + else { + args.extend(["--disable-default-bootnode", "--raw"]); } cmd(binary_path, args).stdout_path(raw_chain_spec).stderr_null().run()?; Ok(raw_chain_spec.to_path_buf()) } -/// Generates a raw chain specification file for a parachain. -/// -/// # Arguments -/// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. -/// * `plain_chain_spec` - Location of the plain chain specification file. -/// * `chain_spec_file_name` - The name of the chain specification file to be generated. -pub fn generate_raw_chain_spec_container_chain( - binary_path: &Path, - raw_chain_spec: &Path, - parachain_id: &str, -) -> Result { - check_command_exists(&binary_path, "build-spec")?; - cmd( - binary_path, - vec!["build-spec", "--disable-default-bootnode", "--parachain-id", parachain_id, "--raw"], - ) - .stderr_null() - .stdout_path(&raw_chain_spec) - .run()?; - Ok(raw_chain_spec.to_path_buf()) -} - /// Export the WebAssembly runtime for the parachain. /// /// # Arguments diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index c3376e7a..016383ac 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -9,7 +9,7 @@ pub enum Error { Aborted, #[error("Anyhow error: {0}")] AnyhowError(#[from] anyhow::Error), - #[error("Missing chain with id: {0}")] + #[error("Missing chain: {0}")] ChainNotFound(String), #[error("{0}")] CommonError(#[from] pop_common::Error), diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 61bee610..792b92a7 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-3.0 -use crate::{ - build::{generate_raw_chain_spec_container_chain, generate_raw_chain_spec_tanssi}, - errors::Error, -}; +use crate::{build::generate_raw_chain_spec_container_chain, errors::Error}; use glob::glob; use indexmap::IndexMap; pub use pop_common::{ @@ -103,20 +100,29 @@ impl Zombienet { .filter_map(|b| b) } - /// Build the needed specs before spawn the network. - pub fn build_specs(&mut self) -> Result<(), Error> { - // This order is important, first container chains that can be more than one, then tanssi-node + /// Build the needed specs before spawn the network containing the container chains. + pub fn build_container_chain_specs(&mut self, parachain_name: &str) -> Result<(), Error> { + let parachain_id = self + .parachains + .iter() + .find(|(_, parachain)| parachain.binary.name() == parachain_name) + .ok_or(Error::ChainNotFound(format!("with name: {parachain_name}")))? + .0; + // This order is important, first container chains that can be more than one, then the parachain let mut container_chains: Vec = Vec::new(); - self.parachains.iter().filter(|(&id, _)| id > 1000).try_for_each( + self.parachains.iter().filter(|(&id, _)| &id > parachain_id).try_for_each( |(id, parachain)| -> Result<(), Error> { container_chains.push( generate_raw_chain_spec_container_chain( parachain.binary.path().as_path(), + None, + None, + &id.to_string(), + None, parachain .chain_spec_path .as_deref() .unwrap_or(Path::new("./chain-spec.json")), - &id.to_string(), )? .display() .to_string(), @@ -124,19 +130,24 @@ impl Zombienet { Ok(()) }, )?; - let tanssi_node = self.parachains.get(&1000).ok_or(Error::ChainNotFound("1000".into()))?; - generate_raw_chain_spec_tanssi( + let tanssi_node = self + .parachains + .get(parachain_id) + .ok_or(Error::ChainNotFound(format!("with id: {parachain_id}")))?; + generate_raw_chain_spec_container_chain( tanssi_node.binary.path().as_path(), tanssi_node.chain.as_deref(), + Some(container_chains.iter().map(|s| s.as_str()).collect()), + ¶chain_id.to_string(), + Some( + tanssi_node + .collators_names + .iter() + .filter(|s| s.starts_with("Collator")) + .map(|s| s.as_str()) + .collect(), + ), tanssi_node.chain_spec_path.as_deref().unwrap_or(Path::new("./chain-spec.json")), - container_chains.iter().map(|s| s.as_str()).collect(), - tanssi_node - .collators_names - .iter() - .filter(|s| s.starts_with("Collator")) - .map(|s| s.as_str()) - .collect(), - &1000.to_string(), )?; Ok(()) } From 5715170449a94084586c2f7c1a6df1d41a90e6e3 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 22:39:35 +0200 Subject: [PATCH 12/19] test: fix unit tests --- crates/pop-parachains/src/up/mod.rs | 51 ++++++++++++++++++++-- crates/pop-parachains/src/up/parachains.rs | 18 ++++++-- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 792b92a7..d6042782 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -1561,6 +1561,7 @@ validator = true fs::{create_dir_all, File}, io::{Read, Write}, path::PathBuf, + vec, }; use tempfile::{tempdir, Builder}; @@ -1711,6 +1712,8 @@ command = "./target/release/parachain-template-node" }, chain: None, chain_spec_generator: None, + chain_spec_path: None, + collators_names: vec!["asset-hub".to_string()], }, ), ( @@ -1724,6 +1727,8 @@ command = "./target/release/parachain-template-node" }, chain: None, chain_spec_generator: None, + chain_spec_path: None, + collators_names: vec!["pop".to_string()], }, ), ( @@ -1737,6 +1742,8 @@ command = "./target/release/parachain-template-node" }, chain: None, chain_spec_generator: None, + chain_spec_path: None, + collators_names: vec!["collator".to_string()], }, ), ] @@ -1866,6 +1873,8 @@ command = "polkadot-parachain" path: system_chain_spec_generator.to_path_buf(), manifest: None, }), + chain_spec_path: None, + collators_names: vec!["asset-hub".to_string()], }, )] .into(), @@ -1944,12 +1953,20 @@ node_spawn_timeout = 300 let name = "parachain-template-node"; let command = PathBuf::from("./target/release").join(&name); assert_eq!( - Parachain::from_local(2000, command.clone(), Some("dev"))?, + Parachain::from_local( + 2000, + command.clone(), + Some("dev"), + Some("/path"), + vec![name] + )?, Parachain { id: 2000, binary: Binary::Local { name: name.to_string(), path: command, manifest: None }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec![name.to_string()], } ); Ok(()) @@ -1960,7 +1977,13 @@ node_spawn_timeout = 300 let name = "pop-parachains"; let command = PathBuf::from("./target/release").join(&name); assert_eq!( - Parachain::from_local(2000, command.clone(), Some("dev"))?, + Parachain::from_local( + 2000, + command.clone(), + Some("dev"), + Some("/path"), + vec![name] + )?, Parachain { id: 2000, binary: Binary::Local { @@ -1970,6 +1993,8 @@ node_spawn_timeout = 300 }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec![name.to_string()], } ); Ok(()) @@ -1980,7 +2005,14 @@ node_spawn_timeout = 300 let repo = Repository::parse("https://git.com/r0gue-io/pop-node#v1.0")?; let cache = tempdir()?; assert_eq!( - Parachain::from_repository(2000, &repo, Some("dev"), cache.path())?, + Parachain::from_repository( + 2000, + &repo, + Some("dev"), + cache.path(), + Some("/path"), + vec!["pop-node"] + )?, Parachain { id: 2000, binary: Binary::Source { @@ -1996,6 +2028,8 @@ node_spawn_timeout = 300 }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec!["pop-node".to_string()], } ); Ok(()) @@ -2006,7 +2040,14 @@ node_spawn_timeout = 300 let repo = Repository::parse("https://github.com/r0gue-io/pop-node#v1.0")?; let cache = tempdir()?; assert_eq!( - Parachain::from_repository(2000, &repo, Some("dev"), cache.path())?, + Parachain::from_repository( + 2000, + &repo, + Some("dev"), + cache.path(), + Some("/path"), + vec!["pop-node"] + )?, Parachain { id: 2000, binary: Binary::Source { @@ -2023,6 +2064,8 @@ node_spawn_timeout = 300 }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_path: Some(PathBuf::from("/path")), + collators_names: vec!["pop-node".to_string()], }, ); Ok(()) diff --git a/crates/pop-parachains/src/up/parachains.rs b/crates/pop-parachains/src/up/parachains.rs index e63c3782..5cbc8230 100644 --- a/crates/pop-parachains/src/up/parachains.rs +++ b/crates/pop-parachains/src/up/parachains.rs @@ -274,9 +274,17 @@ mod tests { let para_id = 2000; let temp_dir = tempdir()?; - let parachain = from(para_id, expected.binary(), Some(version), None, temp_dir.path()) - .await? - .unwrap(); + let parachain = from( + para_id, + expected.binary(), + Some(version), + None, + temp_dir.path(), + Some("/path"), + vec!["pop-node"], + ) + .await? + .unwrap(); assert_eq!(para_id, parachain.id); assert!(matches!(parachain.binary, Binary::Source { name, source, cache } if name == expected.binary() && source == Source::GitHub(ReleaseArchive { @@ -294,7 +302,9 @@ mod tests { #[tokio::test] async fn from_handles_unsupported_command() -> anyhow::Result<()> { - assert!(from(2000, "none", None, None, &PathBuf::default()).await?.is_none()); + assert!(from(2000, "none", None, None, &PathBuf::default(), None, vec!["pop-node"]) + .await? + .is_none()); Ok(()) } } From ee6dfed925c564b0ecba5259949236bb378ef46c Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Tue, 3 Sep 2024 22:41:29 +0200 Subject: [PATCH 13/19] refactor: network.templ for container chains --- .../templates/container/network.templ | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/pop-parachains/templates/container/network.templ b/crates/pop-parachains/templates/container/network.templ index e495c281..fc69b942 100644 --- a/crates/pop-parachains/templates/container/network.templ +++ b/crates/pop-parachains/templates/container/network.templ @@ -20,26 +20,31 @@ validator = true [[parachains]] id = 1000 chain = "dancebox-local" +chain_spec_path = "tanssi-1000.json" default_command = "tanssi-node" [[parachains.collators]] -name = "collator-01" +name = "FullNode-1000" [[parachains.collators]] -name = "collator-02" +name = "Collator1000-01" [[parachains.collators]] -name = "collator-03" +name = "Collator1000-02" [[parachains.collators]] -name = "collator-04" +name = "Collator2000-01" + +[[parachains.collators]] +name = "Collator2000-02" [[parachains]] id = 2000 +chain_spec_path = "template-container-2000.json" default_command = "./target/release/^^node^^" [[parachains.collators]] -name = "full-node-01" +name = "FullNode-2000" [[parachains.collators]] -name = "full-node-02" +name = "FullNode-2000-2" From cdf34dfecc182471e8d51bee4151706c7c96d928 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Wed, 4 Sep 2024 10:56:59 +0200 Subject: [PATCH 14/19] test: unit tests --- crates/pop-parachains/src/build.rs | 88 +++++++++++++++++++- crates/pop-parachains/src/errors.rs | 2 +- crates/pop-parachains/src/new_parachain.rs | 2 +- crates/pop-parachains/src/up/mod.rs | 94 +++++++++++++++++++++- 4 files changed, 179 insertions(+), 7 deletions(-) diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 4d61f829..7430545e 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -356,8 +356,9 @@ impl ChainSpec { mod tests { use super::*; use crate::{ - new_parachain::instantiate_standard_template, templates::Parachain, Config, Error, - Zombienet, + new_parachain::{instantiate_standard_template, instantiate_tanssi_template}, + templates::Parachain, + Config, Error, Zombienet, }; use anyhow::Result; use pop_common::manifest::Dependency; @@ -531,6 +532,89 @@ default_command = "pop-node" Ok(()) } + #[tokio::test] + async fn generate_raw_chain_spec_container_chain_works() -> Result<()> { + let temp_dir = tempdir().expect("Failed to create temp dir"); + instantiate_tanssi_template(&Parachain::TanssiSimple, temp_dir.path(), None)?; + let config = Builder::new().suffix(".toml").tempfile()?; + // Get Zombienet config and fet binary for generate specs + writeln!( + config.as_file(), + "{}", + format!( + r#" +[relaychain] +chain = "paseo-local" + +[[parachains]] +id = 1000 +chain_spec_path = "{}" +chain = "dancebox-local" +default_command = "tanssi-node" + +[[parachains.collators]] +name = "FullNode-1000" + +[[parachains.collators]] +name = "Collator1000-01" + +[[parachains.collators]] +name = "Collator1000-02" + +[[parachains.collators]] +name = "Collator2000-01" + +[[parachains.collators]] +name = "Collator2000-02" +"#, + &temp_dir.path().join("tanssi-1000.json").display().to_string() + ) + )?; + let mut zombienet = Zombienet::new( + &temp_dir.path(), + config.path().to_str().unwrap(), + None, + None, + None, + None, + None, + ) + .await?; + // Fetch the tanssi-node binary + let mut binary_name: String = "".to_string(); + for binary in zombienet.binaries().filter(|b| !b.exists() && b.name() == "tanssi-node") { + binary_name = format!("{}-{}", binary.name(), binary.latest().unwrap()); + binary.source(true, &(), true).await?; + } + + let raw_chain_spec_container_chain = generate_raw_chain_spec_container_chain( + &temp_dir.path().join(binary_name.clone()), + None, + None, + &"2000", + None, + &temp_dir.path().join("raw-container-chain-chainspec.json"), + )?; + assert!(raw_chain_spec_container_chain.exists()); + + let raw_chain_spec = generate_raw_chain_spec_container_chain( + &temp_dir.path().join(binary_name), + Some("dancebox-local"), + Some(vec![&temp_dir + .path() + .join("raw-container-chain-chainspec.json") + .display() + .to_string()]), + &"1000", + Some(vec!["Collator1000-01", "Collator1000-02", "Collator2000-01", "Collator2000-02"]), + &temp_dir.path().join("raw-parachain-chainspec.json"), + )?; + assert!(raw_chain_spec.exists()); + let content = fs::read_to_string(raw_chain_spec.clone()).expect("Could not read file"); + assert!(content.contains("\"para_id\": 2000")); + Ok(()) + } + #[test] fn raw_chain_spec_fails_wrong_chain_spec() -> Result<()> { assert!(matches!( diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index 016383ac..0e3fee1c 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -9,7 +9,7 @@ pub enum Error { Aborted, #[error("Anyhow error: {0}")] AnyhowError(#[from] anyhow::Error), - #[error("Missing chain: {0}")] + #[error("Missing chain {0}")] ChainNotFound(String), #[error("{0}")] CommonError(#[from] pop_common::Error), diff --git a/crates/pop-parachains/src/new_parachain.rs b/crates/pop-parachains/src/new_parachain.rs index 33a9c305..4c4bdf92 100644 --- a/crates/pop-parachains/src/new_parachain.rs +++ b/crates/pop-parachains/src/new_parachain.rs @@ -94,7 +94,7 @@ pub fn instantiate_standard_template( /// * `template` - template to generate the container-chain from. /// * `target` - location where the parachain will be created. /// * `tag_version` - version to use (`None` to use latest). -fn instantiate_tanssi_template( +pub fn instantiate_tanssi_template( template: &ContainerChain, target: &Path, tag_version: Option, diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index d6042782..21f3a1fa 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -101,7 +101,7 @@ impl Zombienet { } /// Build the needed specs before spawn the network containing the container chains. - pub fn build_container_chain_specs(&mut self, parachain_name: &str) -> Result<(), Error> { + pub fn build_container_chain_specs(&mut self, parachain_name: &str) -> Result { let parachain_id = self .parachains .iter() @@ -134,7 +134,7 @@ impl Zombienet { .parachains .get(parachain_id) .ok_or(Error::ChainNotFound(format!("with id: {parachain_id}")))?; - generate_raw_chain_spec_container_chain( + let raw_spec_path = generate_raw_chain_spec_container_chain( tanssi_node.binary.path().as_path(), tanssi_node.chain.as_deref(), Some(container_chains.iter().map(|s| s.as_str()).collect()), @@ -149,7 +149,7 @@ impl Zombienet { ), tanssi_node.chain_spec_path.as_deref().unwrap_or(Path::new("./chain-spec.json")), )?; - Ok(()) + Ok(raw_spec_path) } /// Determine parachain configuration based on specified version and network configuration. @@ -767,11 +767,16 @@ fn resolve_manifest(package: &str, path: &Path) -> Result, Error #[cfg(test)] mod tests { use super::*; + use crate::{ + new_parachain::instantiate_tanssi_template, templates::Parachain as Template, Error, + }; use anyhow::Result; use std::{env::current_dir, fs::File, io::Write}; use tempfile::tempdir; mod zombienet { + use std::fs; + use super::*; use pop_common::Status; @@ -1415,6 +1420,89 @@ chain = "asset-hub-paseo-local" Ok(()) } + #[tokio::test] + async fn build_container_chain_specs_works() -> Result<()> { + let temp_dir = tempdir().expect("Failed to create temp dir"); + instantiate_tanssi_template(&Template::TanssiSimple, temp_dir.path(), None)?; + let config = Builder::new().suffix(".toml").tempfile()?; + // Get Zombienet config and fet binary for generate specs + writeln!( + config.as_file(), + "{}", + format!( + r#" +[relaychain] +chain = "paseo-local" + +[[parachains]] +id = 1000 +chain_spec_path = "{}" +chain = "dancebox-local" +default_command = "tanssi-node" + +[[parachains.collators]] +name = "FullNode-1000" + +[[parachains.collators]] +name = "Collator1000-01" + +[[parachains.collators]] +name = "Collator1000-02" + +[[parachains.collators]] +name = "Collator2000-01" + +[[parachains.collators]] +name = "Collator2000-02" +"#, + &temp_dir.path().join("tanssi-1000.json").display().to_string() + ) + )?; + let mut zombienet = Zombienet::new( + &temp_dir.path(), + config.path().to_str().unwrap(), + None, + None, + None, + None, + None, + ) + .await?; + // Fetch the tanssi-node binary + for binary in zombienet.binaries().filter(|b| !b.exists() && b.name() == "tanssi-node") + { + binary.source(true, &(), true).await?; + } + let raw_chain_spec = zombienet.build_container_chain_specs("tanssi-node")?; + assert!(raw_chain_spec.exists()); + let content = fs::read_to_string(raw_chain_spec.clone()).expect("Could not read file"); + assert!(content.contains("\"para_id\": 1000")); + assert!(content.contains("\"id\": \"dancebox_local\"")); + Ok(()) + } + + #[tokio::test] + async fn build_container_chain_specs_fails_chain_not_found() -> Result<()> { + let temp_dir = tempdir().expect("Failed to create temp dir"); + instantiate_tanssi_template(&Template::TanssiSimple, temp_dir.path(), None)?; + let mut zombienet = Zombienet::new( + &temp_dir.path(), + &temp_dir.path().join("network.toml").display().to_string(), + None, + None, + None, + None, + None, + ) + .await?; + assert!(matches!( + zombienet.build_container_chain_specs("bad-name"), + Err(Error::ChainNotFound(error)) + if error == "with name: bad-name" + )); + Ok(()) + } + #[tokio::test] async fn spawn_ensures_relay_chain_binary_exists() -> Result<()> { let temp_dir = tempdir()?; From 62811d24ab9008bef4fbba48c20cf09d57e3e830 Mon Sep 17 00:00:00 2001 From: AlexD10S Date: Wed, 4 Sep 2024 16:42:55 +0200 Subject: [PATCH 15/19] fix: clone_and_degit to fetch from latest tag instead of master --- crates/pop-common/src/git.rs | 42 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/crates/pop-common/src/git.rs b/crates/pop-common/src/git.rs index f93fbc14..b6fecb0f 100644 --- a/crates/pop-common/src/git.rs +++ b/crates/pop-common/src/git.rs @@ -66,32 +66,38 @@ impl Git { target, )?, }; - + // Checkout the specific tag if provided if let Some(tag_version) = tag_version { - let (object, reference) = repo.revparse_ext(&tag_version).expect("Object not found"); - repo.checkout_tree(&object, None).expect("Failed to checkout"); - match reference { - // gref is an actual reference like branches or tags - Some(gref) => repo.set_head(gref.name().unwrap()), - // this is a commit, not a reference - None => repo.set_head_detached(object.id()), - } - .expect("Failed to set HEAD"); - - let git_dir = repo.path(); - fs::remove_dir_all(&git_dir)?; - return Ok(Some(tag_version)); + return Self::git_checkout(&repo, tag_version); } - - // fetch tags from remote + // If not to the latest release let release = Self::fetch_latest_tag(&repo); - + if let Some(tag_version) = release { + return Self::git_checkout(&repo, tag_version); + } + // If no tags, take it from main branch let git_dir = repo.path(); fs::remove_dir_all(&git_dir)?; - // Or by default the last one Ok(release) } + // Checkout to a specific tag or commit from a Git repository. + fn git_checkout(repo: &GitRepository, tag_version: String) -> Result> { + let (object, reference) = repo.revparse_ext(&tag_version).expect("Object not found"); + repo.checkout_tree(&object, None).expect("Failed to checkout"); + match reference { + // gref is an actual reference like branches or tags + Some(gref) => repo.set_head(gref.name().unwrap()), + // this is a commit, not a reference + None => repo.set_head_detached(object.id()), + } + .expect("Failed to set HEAD"); + + let git_dir = repo.path(); + fs::remove_dir_all(&git_dir)?; + Ok(Some(tag_version)) + } + /// For users that have ssh configuration for cloning repositories. fn ssh_clone_and_degit(url: Url, target: &Path) -> Result { let ssh_url = GitHub::convert_to_ssh_url(&url); From 8bc15a93671f2c049d6e6d60a5f23c1f7d310517 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Wed, 4 Sep 2024 20:13:15 +0200 Subject: [PATCH 16/19] refactor clone°it --- crates/pop-common/src/git.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/pop-common/src/git.rs b/crates/pop-common/src/git.rs index b6fecb0f..b6b582c6 100644 --- a/crates/pop-common/src/git.rs +++ b/crates/pop-common/src/git.rs @@ -66,19 +66,21 @@ impl Git { target, )?, }; - // Checkout the specific tag if provided - if let Some(tag_version) = tag_version { - return Self::git_checkout(&repo, tag_version); - } - // If not to the latest release - let release = Self::fetch_latest_tag(&repo); - if let Some(tag_version) = release { - return Self::git_checkout(&repo, tag_version); + + match tag_version { + Some(tag) => Self::git_checkout(&repo, tag), + None => { + // Fetch latest release. + let release = Self::fetch_latest_tag(&repo); + if let Some(r) = release { + return Self::git_checkout(&repo, r); + } + // If there are no releases, use main. + let git_dir = repo.path(); + fs::remove_dir_all(&git_dir)?; + Ok(release) + }, } - // If no tags, take it from main branch - let git_dir = repo.path(); - fs::remove_dir_all(&git_dir)?; - Ok(release) } // Checkout to a specific tag or commit from a Git repository. From 00a6c0e1a20afea843e1ebe14617300fb5941faa Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:04:12 +0200 Subject: [PATCH 17/19] chore: remove evm container --- crates/pop-parachains/src/templates.rs | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/crates/pop-parachains/src/templates.rs b/crates/pop-parachains/src/templates.rs index 479ccaca..9b01f3bc 100644 --- a/crates/pop-parachains/src/templates.rs +++ b/crates/pop-parachains/src/templates.rs @@ -181,23 +181,6 @@ pub enum Parachain { ) )] TanssiSimple, - /// Tanssi EVM enabled container chain template. - #[strum( - serialize = "frontier", - message = "EVM", - detailed_message = "EVM enabled container chain template.", - props( - Provider = "Tanssi", - Repository = "https://github.com/moondance-labs/tanssi", - Network = "./network.toml", - NodeDirectory = "container-chains/nodes", - RuntimeDirectory = "container-chains/runtime-templates", - SupportedVersions = "master", - ExtractableFiles = ".rustfmt.toml,Cargo.toml,Cargo.lock,LICENSE,README.md,rust-toolchain", - ) - )] - TanssiFrontier, - // Templates for unit tests below. #[cfg(test)] #[strum( @@ -292,7 +275,6 @@ mod tests { ("fpt".to_string(), ParityFPT), // Tanssi. ("simple".to_string(), TanssiSimple), - ("frontier".to_string(), TanssiFrontier), // Test. ("test_01".to_string(), TestTemplate01), ("test_02".to_string(), TestTemplate02), @@ -316,7 +298,6 @@ mod tests { ("fpt".to_string(), "https://github.com/paritytech/frontier-parachain-template"), // Tanssi. ("simple".to_string(), "https://github.com/moondance-labs/tanssi"), - ("frontier".to_string(), "https://github.com/moondance-labs/tanssi"), // Test. ("test_01".to_string(), ""), ("test_02".to_string(), ""), @@ -337,7 +318,6 @@ mod tests { (ParityFPT, Some("./zombienet-config.toml")), // Tanssi. (TanssiSimple, Some("./network.toml")), - (TanssiFrontier, Some("./network.toml")), // Test. (TestTemplate01, Some("")), (TestTemplate02, Some("")), @@ -366,7 +346,7 @@ mod tests { assert_eq!(Provider::OpenZeppelin.provides(&template), true); assert_eq!(Provider::Tanssi.provides(&template), false); } - if matches!(template, TanssiSimple | TanssiFrontier) { + if matches!(template, TanssiSimple ) { assert_eq!(Provider::Pop.provides(&template), false); assert_eq!(Provider::Parity.provides(&template), false); assert_eq!(Provider::OpenZeppelin.provides(&template), false); From 406e301ab872192f89689e6421ea7d56e2f1d91b Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:19:11 +0200 Subject: [PATCH 18/19] cargo +nightly fmt --- crates/pop-cli/src/commands/up/parachain.rs | 3 ++- crates/pop-parachains/src/build.rs | 12 ++++++++---- crates/pop-parachains/src/templates.rs | 2 +- crates/pop-parachains/src/up/mod.rs | 3 ++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/pop-cli/src/commands/up/parachain.rs b/crates/pop-cli/src/commands/up/parachain.rs index a80efb61..510d638a 100644 --- a/crates/pop-cli/src/commands/up/parachain.rs +++ b/crates/pop-cli/src/commands/up/parachain.rs @@ -90,7 +90,8 @@ impl ZombienetCommand { return Ok(()); } // For container chains, build the specs before spawn the network - // TODO: Is there a better way to identifiy it than to check if there is a hardcoded tanssi-node + // TODO: Is there a better way to identifiy it than to check if there is a hardcoded + // tanssi-node if zombienet.binaries().any(|b| b.name() == "tanssi-node") { zombienet.build_container_chain_specs("tanssi-node")?; } diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build.rs index 7430545e..2b18fd33 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build.rs @@ -125,11 +125,15 @@ pub fn generate_raw_chain_spec( /// /// # Arguments /// * `binary_path` - The path to the node binary executable that contains the `build-spec` command. -/// * `chain` - An optional chain to be used. This should be provided if generating a spec for a parachain. -/// * `container_chains` - An optional vector of container chain specification files. These should be provided if generating a spec for a parachain. +/// * `chain` - An optional chain to be used. This should be provided if generating a spec for a +/// parachain. +/// * `container_chains` - An optional vector of container chain specification files. These should +/// be provided if generating a spec for a parachain. /// * `id` - The parachain id for which the chain specification is being generated. -/// * `invulnerables` - An optional vector of invulnerable nodes. These should be provided if generating a spec for a parachain. -/// * `raw_chain_spec` - The path where the generated raw chain specification file will be generated. +/// * `invulnerables` - An optional vector of invulnerable nodes. These should be provided if +/// generating a spec for a parachain. +/// * `raw_chain_spec` - The path where the generated raw chain specification file will be +/// generated. pub fn generate_raw_chain_spec_container_chain( binary_path: &Path, chain: Option<&str>, diff --git a/crates/pop-parachains/src/templates.rs b/crates/pop-parachains/src/templates.rs index 5b30cdc1..484ba63d 100644 --- a/crates/pop-parachains/src/templates.rs +++ b/crates/pop-parachains/src/templates.rs @@ -372,7 +372,7 @@ mod tests { assert_eq!(Provider::OpenZeppelin.provides(&template), true); assert_eq!(Provider::Tanssi.provides(&template), false); } - if matches!(template, TanssiSimple ) { + if matches!(template, TanssiSimple) { assert_eq!(Provider::Pop.provides(&template), false); assert_eq!(Provider::Parity.provides(&template), false); assert_eq!(Provider::OpenZeppelin.provides(&template), false); diff --git a/crates/pop-parachains/src/up/mod.rs b/crates/pop-parachains/src/up/mod.rs index 21f3a1fa..71738c05 100644 --- a/crates/pop-parachains/src/up/mod.rs +++ b/crates/pop-parachains/src/up/mod.rs @@ -108,7 +108,8 @@ impl Zombienet { .find(|(_, parachain)| parachain.binary.name() == parachain_name) .ok_or(Error::ChainNotFound(format!("with name: {parachain_name}")))? .0; - // This order is important, first container chains that can be more than one, then the parachain + // This order is important, first container chains that can be more than one, then the + // parachain let mut container_chains: Vec = Vec::new(); self.parachains.iter().filter(|(&id, _)| &id > parachain_id).try_for_each( |(id, parachain)| -> Result<(), Error> { From c032c0d51d93e81dfdbfd11a41f08754f1758bf3 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Andres <11448715+al3mart@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:03:04 +0200 Subject: [PATCH 19/19] fix: revert extractor changes --- crates/pop-common/src/templates/extractor.rs | 30 +++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/crates/pop-common/src/templates/extractor.rs b/crates/pop-common/src/templates/extractor.rs index 5f801999..95ee2c86 100644 --- a/crates/pop-common/src/templates/extractor.rs +++ b/crates/pop-common/src/templates/extractor.rs @@ -17,14 +17,24 @@ pub fn extract_template_files( target_directory: &Path, ignore_directories: Option>, ) -> Result<()> { - let template_directory = repo_directory.join(template_name); - // Recursively copy all directories and files within. Ignores the specified ones. - copy_dir_all( - &template_directory, - target_directory, - &ignore_directories.unwrap_or_else(|| vec![]), - )?; - Ok(()) + let template_directory = repo_directory.join(&template_name); + if template_directory.is_dir() { + // Recursively copy all directories and files within. Ignores the specified ones. + copy_dir_all( + &template_directory, + target_directory, + &ignore_directories.unwrap_or_else(|| vec![]), + )?; + return Ok(()); + } else { + // If not a dir, just copy the file. + let dst = target_directory.join(&template_name); + // In case the first file being pulled is not a directory, + // Make sure the target directory exists. + fs::create_dir_all(&target_directory)?; + fs::copy(template_directory, &dst)?; + Ok(()) + } } /// Recursively copy a directory and its files. @@ -42,8 +52,8 @@ fn copy_dir_all( for entry in fs::read_dir(src)? { let entry = entry?; let ty = entry.file_type()?; - if ty.is_dir() && - ignore_directories.contains(&entry.file_name().to_string_lossy().to_string()) + if ty.is_dir() + && ignore_directories.contains(&entry.file_name().to_string_lossy().to_string()) { continue; } else if ty.is_dir() {