feat: Move bootstrap node into collator

Justfile

@@ -24,8 +24,10 @@ release: lint
 release-testnet:
     cargo build --release --features polka-storage-runtime/testnet --bin polka-storage-node
 
-# Run the testnet without building
+# Generate a private key for the P2P network and
+# run the testnet without building
 run-testnet:
+    openssl genpkey -algorithm ED25519 -out /tmp/polka-storage/private.pem
     zombienet -p native spawn zombienet/local-testnet.toml
 
 # Run the testing building it before
@@ -152,6 +154,7 @@ load-to-minikube:
     minikube image load ghcr.io/polka-storage-node:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | select(.name == "polka-storage-node") | .version')"
 
 kube-testnet:
+    openssl genpkey -algorithm ED25519 -out /tmp/polka-storage/private.pem
     zombienet -p kubernetes spawn zombienet/local-kube-testnet.toml
 
 # The tarpaulin calls for test coverage have the following options:

docs/src/storage-provider-cli/server.md

@@ -5,6 +5,8 @@ This chapter covers the Polka Storage Provider server.
 ## P2P Key generation
 
 The Polka Storage Provider server runs a p2p node that is used to map Peer ID's to Multi-addresses.
+The generated Peer ID is used for on-chain registration and inside the P2P network.
+Both these Peer ID's must be the same.
 The server can run either a bootstrap node, used to aid in discovery,
 or a registration node, which registers to a bootstrap node with their Peer ID to Multi-address mapping.
 For both of these node types, the server needs an ed25519 private key.

examples/rpc_publish.sh

@@ -37,6 +37,9 @@ openssl genpkey -algorithm ED25519 -out "$P2P_PRIVATE_KEY"
 # https://github.com/openssl/openssl/commit/6c03fa21ed4bbc9fd6d3013fdf9f4646d231f831
 openssl pkey -in "$P2P_PRIVATE_KEY" -pubout -out "$P2P_PUBLIC_KEY"
 
+# Generate Peer ID
+P2P_BOOTSTRAP_PEER_ID="$(target/release/polka-storage-provider-client generate-peer-id --pubkey "$P2P_PUBLIC_KEY")"
+
 # Convert file to CARv2 format
 target/release/mater-cli convert -q --overwrite "$INPUT_FILE" "$INPUT_TMP_FILE" &&
 
@@ -54,7 +57,8 @@ post_proof = '2KiB'
 porep_parameters = '2KiB.porep.params'
 post_parameters = '2KiB.post.params'
 rendezvous_point_address = '$P2P_ADDRESS'
-p2p_key = '@$P2P_PRIVATE_KEY'" > "$CONFIG"
+p2p_key = '@$P2P_PRIVATE_KEY'
+rendezvous_point = '$P2P_BOOTSTRAP_PEER_ID'" > "$CONFIG"
 
 
 # Setup balances

maat/Cargo.toml

@@ -23,6 +23,7 @@ bls12_381 = { workspace = true }
 bs58.workspace = true
 cid.workspace = true
 codec = { workspace = true }
+ed25519-dalek = { workspace = true, features = ["pem"] }
 futures.workspace = true
 hex = { workspace = true }
 libp2p = { workspace = true, features = ["ecdsa", "identify"] }
@@ -33,6 +34,7 @@ serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true, features = ["std"] }
 storagext = { workspace = true }
 subxt = { workspace = true, features = ["substrate-compat"] }
+tempfile = { workspace = true }
 thiserror.workspace = true
 tokio = { workspace = true, features = ["full"] }
 tokio-util = { workspace = true, features = ["rt"] }
@@ -43,8 +45,5 @@ zombienet-configuration.workspace = true
 zombienet-sdk.workspace = true
 zombienet-support.workspace = true
 
-[dev-dependencies]
-tempfile = { workspace = true }
-
 [lints]
 workspace = true

maat/src/lib.rs

@@ -1,5 +1,13 @@
-use std::path::PathBuf;
+use std::{
+    fs::File,
+    io::Write,
+    path::{Path, PathBuf},
+};
 
+use ed25519_dalek::{
+    pkcs8::{spki::der::pem::LineEnding, EncodePrivateKey},
+    SigningKey,
+};
 use storagext::PolkaStorageConfig;
 use subxt::{
     ext::{
@@ -8,6 +16,7 @@ use subxt::{
     },
     tx::PairSigner,
 };
+use tempfile::tempdir;
 use tracing::level_filters::LevelFilter;
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
 use zombienet_configuration::shared::node::{Buildable, Initial, NodeConfigBuilder};
@@ -92,6 +101,9 @@ pub fn local_testnet_config() -> NetworkConfig {
         .display()
         .to_string();
     let polka_storage_node_binary_path = binding.as_str();
+    let temp_dir = tempdir().unwrap();
+    let file_path = temp_dir.path().join("private_key.pem");
+    generate_pem_file(&file_path);
 
     NetworkConfigBuilder::new()
         .with_relaychain(|relaychain| {
@@ -111,6 +123,8 @@ pub fn local_testnet_config() -> NetworkConfig {
                         .with_args(vec![
                             ("--pool-type", "fork-aware").into(),
                             ("-lruntime=trace,parachain=debug").into(),
+                            ("--p2p-listen-address=/ip4/127.0.0.1/tcp/62649").into(),
+                            (format!("--p2p-key={}", file_path.display()).as_str()).into(),
                         ])
                 })
         })
@@ -138,3 +152,10 @@ where
     let keypair = Pair::from_string(s, None).unwrap();
     PairSigner::<PolkaStorageConfig, P>::new(keypair)
 }
+
+fn generate_pem_file<P: AsRef<Path>>(path: P) {
+    let signing_key = SigningKey::from([0; 32]);
+    let pem = signing_key.to_pkcs8_pem(LineEnding::default()).unwrap();
+    let mut file = File::create(path).unwrap();
+    write!(file, "{}", *pem).unwrap();
+}

node/Cargo.toml

@@ -15,6 +15,7 @@ name = "polka-storage-node"
 
 [dependencies]
 polka-storage-runtime.workspace = true
+primitives = { workspace = true, features = ["std"] }
 
 clap = { features = ["derive"], workspace = true }
 codec = { workspace = true, default-features = true }
@@ -33,6 +34,7 @@ frame-benchmarking = { workspace = true, default-features = true }
 frame-benchmarking-cli = { workspace = true, default-features = true }
 futures = { workspace = true }
 jsonrpsee = { features = ["server"], workspace = true }
+libp2p = { workspace = true, features = ["identify", "macros", "noise", "rendezvous", "tcp", "tokio", "yamux"] }
 log = { workspace = true, default-features = true }
 pallet-transaction-payment-rpc = { workspace = true, default-features = true }
 polkadot-cli = { features = ["rococo-native"], workspace = true, default-features = true }
@@ -66,6 +68,7 @@ sp-keystore = { workspace = true, default-features = true }
 sp-runtime = { workspace = true, default-features = true }
 sp-timestamp = { workspace = true, default-features = true }
 substrate-frame-rpc-system = { workspace = true, default-features = true }
+thiserror = { workspace = true }
 xcm.workspace = true
 
 [build-dependencies]

node/src/cli.rs

@@ -1,5 +1,8 @@
 use std::path::PathBuf;
 
+use libp2p::{identity::Keypair, Multiaddr};
+use primitives::p2p::keypair_value_parser;
+
 /// Sub-commands supported by the collator.
 #[derive(Debug, clap::Subcommand)]
 #[allow(clippy::large_enum_variant)]
@@ -64,7 +67,7 @@ pub struct Cli {
     pub subcommand: Option<Subcommand>,
 
     #[command(flatten)]
-    pub run: cumulus_client_cli::RunCmd,
+    pub run: RunCmd,
 
     /// Disable automatic hardware benchmarks.
     ///
@@ -109,3 +112,21 @@ impl RelayChainCli {
         }
     }
 }
+
+#[derive(Debug, clap::Parser)]
+#[group(skip)]
+pub struct RunCmd {
+    #[clap(flatten)]
+    pub base: cumulus_client_cli::RunCmd,
+
+    /// Key used in the P2P network of Storage Providers and Collators.
+    /// This key generates the Peer ID that storage providers use to register.
+    /// It must be an ED25519 private key, either in PEM format or passed in directly.
+    #[arg(long, value_parser = keypair_value_parser, required = false)]
+    pub p2p_key: Option<Keypair>,
+
+    /// Listen address in the P2P network of Storage Providers and Collators
+    /// that the bootstrap node binds to.
+    #[arg(long, required = false)]
+    pub p2p_listen_address: Option<Multiaddr>,
+}

node/src/command.rs

@@ -12,7 +12,7 @@ use sc_service::config::{BasePath, PrometheusConfig};
 use crate::{
     chain_spec,
     cli::{Cli, RelayChainCli, Subcommand},
-    service::new_partial,
+    service::{new_partial, p2p::BootstrapConfig},
 };
 
 fn load_spec(id: &str) -> std::result::Result<Box<dyn ChainSpec>, String> {
@@ -221,10 +221,28 @@ pub fn run() -> Result<()> {
             }
         }
         None => {
-            let runner = cli.create_runner(&cli.run.normalize())?;
-            let collator_options = cli.run.collator_options();
+            let runner = cli.create_runner(&cli.run.base.normalize())?;
+            let collator_options = cli.run.base.collator_options();
 
             runner.run_node_until_exit(|config| async move {
+                let bootstrap_config = if config.role.is_authority() {
+                    let p2p_key = cli
+                        .run
+                        .p2p_key
+                        .ok_or(
+                            "This node is configured as authority, so it will be used as Bootstrap node for Storage Provider & Collator network, but the key is missing. Set the --p2p-key argument of the node."
+                        )?;
+                    let p2p_listen_address = cli
+                        .run
+                        .p2p_listen_address
+                        .ok_or(
+                            "This node is configured as authority, so it will be used as Bootstrap node for Storage Provider & Collator network, but the listen address is missing. Set the --p2p-listen-address argument of the node."
+                        )?;
+                    Some(BootstrapConfig::new(p2p_key, p2p_listen_address))
+                } else {
+                    None
+                };
+
                 let hwbench = (!cli.no_hardware_benchmarks)
                     .then_some(config.database.path().map(|database_path| {
                         let _ = std::fs::create_dir_all(database_path);
@@ -268,6 +286,7 @@ pub fn run() -> Result<()> {
                     collator_options,
                     id,
                     hwbench,
+                    bootstrap_config,
                 )
                 .await
                 .map(|r| r.0)

node/src/service.rs → node/src/service/mod.rs

@@ -38,6 +38,10 @@ use sc_telemetry::{Telemetry, TelemetryHandle, TelemetryWorker, TelemetryWorkerH
 use sc_transaction_pool_api::OffchainTransactionPoolFactory;
 use sp_keystore::KeystorePtr;
 
+pub mod p2p;
+
+use crate::service::p2p::{run_bootstrap_node, BootstrapConfig};
+
 #[docify::export(wasm_executor)]
 type ParachainExecutor = WasmExecutor<ParachainHostFunctions>;
 
@@ -245,6 +249,7 @@ pub async fn start_parachain_node(
     collator_options: CollatorOptions,
     para_id: ParaId,
     hwbench: Option<sc_sysinfo::HwBench>,
+    bootstrap_config: Option<BootstrapConfig>,
 ) -> sc_service::error::Result<(TaskManager, Arc<ParachainClient>)> {
     let parachain_config = prepare_node_config(parachain_config);
 
@@ -332,6 +337,12 @@ pub async fn start_parachain_node(
         })
     };
 
+    if let Some(config) = bootstrap_config {
+        task_manager
+            .spawn_handle()
+            .spawn("p2p", None, run_bootstrap_node(config));
+    }
+
     sc_service::spawn_tasks(sc_service::SpawnTasksParams {
         rpc_builder,
         client: client.clone(),

storage-provider/server/src/p2p/bootstrap.rs → node/src/service/p2p/bootstrap.rs

@@ -8,9 +8,9 @@ use libp2p::{
     swarm::{NetworkBehaviour, SwarmEvent},
     tcp, yamux, Multiaddr, Swarm, SwarmBuilder,
 };
+use log::{debug, info};
 
-use super::P2PError;
-use crate::config::DEFAULT_REGISTRATION_TTL;
+use crate::service::p2p::{P2PError, DEFAULT_REGISTRATION_TTL};
 
 #[derive(NetworkBehaviour)]
 pub struct BootstrapBehaviour {
@@ -62,21 +62,19 @@ pub(crate) async fn bootstrap(
     mut swarm: Swarm<BootstrapBehaviour>,
     addr: Multiaddr,
 ) -> Result<(), P2PError> {
-    tracing::info!("Starting P2P bootstrap node at {addr}");
+    info!("Starting P2P bootstrap node at {addr}");
     swarm.listen_on(addr)?;
-    while let Some(event) = swarm.next().await {
-        match event {
+    loop {
+        match swarm.select_next_some().await {
             SwarmEvent::NewListenAddr { address, .. } => {
-                tracing::info!("Listening on {}", address);
+                info!("Listening on {}", address);
             }
             SwarmEvent::Behaviour(BootstrapBehaviourEvent::Rendezvous(
                 rendezvous::server::Event::PeerRegistered { peer, registration },
             )) => {
-                tracing::info!(
+                info!(
                     "Peer {} registered for namespace '{}' for {} seconds",
-                    peer,
-                    registration.namespace,
-                    registration.ttl
+                    peer, registration.namespace, registration.ttl
                 );
             }
             SwarmEvent::Behaviour(BootstrapBehaviourEvent::Rendezvous(
@@ -86,7 +84,7 @@ pub(crate) async fn bootstrap(
                 },
             )) => {
                 if !registrations.is_empty() {
-                    tracing::info!(
+                    info!(
                         "Served peer {} with {} new registrations",
                         enquirer,
                         registrations.len()
@@ -96,14 +94,13 @@ pub(crate) async fn bootstrap(
             SwarmEvent::Behaviour(BootstrapBehaviourEvent::Rendezvous(
                 rendezvous::server::Event::RegistrationExpired(registration),
             )) => {
-                tracing::info!(
+                info!(
                     "Registration for peer {} expired in namespace {}",
                     registration.record.peer_id(),
                     registration.namespace
                 );
             }
-            other => tracing::debug!("Encountered event: {other:?}"),
+            other => debug!("Encountered event: {other:?}"),
         }
     }
-    Ok(())
 }