feat: Support for targetting arbitrary IC network on the cli (#302)

* WIP * More updates * Finished refactoring * clippy * Some fixes after merging * Add tests and fix some encountered issues * bazel repin * Remove unused struct * Fix bazel build of slack-notifications * Allow network access from bazel tests * Cleanup confusingly duplicate names * Ignore failures of the staging IC test runs, due to firewall restrictions * fix ic-admin args * Prefer private pem key if provided, for trustworthy metrics
dfinity · Apr 8, 2024 · d1ac15b · d1ac15b
1 parent caec95f
commit d1ac15b
Show file tree

Hide file tree

Showing 46 changed files with 781 additions and 427 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -26,7 +26,7 @@ build:lint --config=clippy
 
 build:ci --progress_report_interval=30
 
-build --nosandbox_default_allow_network
+build --sandbox_default_allow_network
 build --incompatible_strict_action_env # use an environment with a static value for PATH and do not inherit LD_LIBRARY_PATH
 
 # default to optimized and unstripped binaries.

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,5 +1,6 @@
 {
     "yaml.schemas": {
         "release-index-shema.json": "release-index.yaml"
-    }
+    },
+    "rust-analyzer.showUnlinkedFileNotification": false
 }
diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock
@@ -1,5 +1,5 @@
 {
-  "checksum": "31058f9f66529f2b114e4c4508e3cc6dbf7243335ca17851bca859ae720bcccd",
+  "checksum": "b6737f3f6bdb970486578a3c4c615afe6fd16e0bf32c3930b649d568c6be06c5",
   "crates": {
     "actix-codec 0.5.2": {
       "name": "actix-codec",
@@ -22116,6 +22116,10 @@
               "id": "chrono 0.4.37",
               "target": "chrono"
             },
+            {
+              "id": "futures 0.3.30",
+              "target": "futures"
+            },
             {
               "id": "ic-base-types 0.9.0",
               "target": "ic_base_types"
@@ -22152,13 +22156,26 @@
               "id": "strum 0.26.2",
               "target": "strum"
             },
+            {
+              "id": "tokio 1.36.0",
+              "target": "tokio"
+            },
             {
               "id": "url 2.5.0",
               "target": "url"
             }
           ],
           "selects": {}
         },
+        "deps_dev": {
+          "common": [
+            {
+              "id": "wiremock 0.6.0",
+              "target": "wiremock"
+            }
+          ],
+          "selects": {}
+        },
         "edition": "2021",
         "proc_macro_deps": {
           "common": [

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -175,6 +175,7 @@ tokio = { version = "1.2.0", features = ["full"] }
 url = "2.5.0"
 urlencoding = "2.1.0"
 warp = "0.3"
+wiremock = "0.6.0"
 
 
 [profile.release]

diff --git a/dashboard/package.json b/dashboard/package.json
@@ -7,9 +7,9 @@
   },
   "scripts": {
     "dev-staging": "concurrently \"yarn start\" \"yarn start-backend\" \"yarn start-rust-backend-staging\"",
-    "start-rust-backend-staging": "cd .. && BACKEND_PORT=8081 RUST_BACKTRACE=1 NETWORK=staging NNS_URL=http://[2600:3000:6100:200:5000:b0ff:fe8e:6b7b]:8080 cargo watch -C rs/ic-management-backend -x 'run --release --color=always'",
+    "start-rust-backend-staging": "cd .. && BACKEND_PORT=8081 RUST_BACKTRACE=1 cargo watch -C rs/ic-management-backend -x 'run --release --color=always -- --network staging'",
     "dev-mainnet": "concurrently \"yarn start\" \"yarn start-backend\" \"yarn start-rust-backend-mainnet\"",
-    "start-rust-backend-mainnet": "cd .. && RUST_BACKTRACE=1 NETWORK=mainnet NNS_URL=https://ic0.app cargo watch -C rs/ic-management-backend -x 'run --release --color=always'",
+    "start-rust-backend-mainnet": "cd .. && RUST_BACKTRACE=1 cargo watch -C rs/ic-management-backend -x 'run --release --color=always -- --network mainnet'",
     "dev": "concurrently \"yarn start\" \"yarn start-backend\" \"yarn start-rust-backend-staging\" \"yarn start-rust-backend-mainnet\"",
     "start": "yarn workspace app start",
     "start-backend": "yarn workspace backend start",
@@ -71,4 +71,4 @@
     ]
   },
   "dependencies": {}
-}
+}
diff --git a/rs/cli/Cargo.toml b/rs/cli/Cargo.toml
@@ -60,7 +60,7 @@ url = { workspace = true }
 tempfile = "3.10.0"
 
 [dev-dependencies]
-wiremock = "0.6.0"
+wiremock = { workspace = true }
 
 [[bin]]
 name = "dre"

diff --git a/rs/cli/src/cli.rs b/rs/cli/src/cli.rs
@@ -4,9 +4,9 @@ use clap::{Parser, Subcommand};
 use clap_num::maybe_hex;
 use ic_base_types::PrincipalId;
 use ic_management_types::{Artifact, Network};
-use log::error;
+use url::Url;
 
-use crate::detect_neuron::{detect_hsm_auth, detect_neuron, Auth, Neuron};
+use crate::detect_neuron::Neuron;
 
 // For more info about the version setup, look at https://docs.rs/clap/latest/clap/struct.Command.html#method.version
 #[derive(Parser, Clone, Default)]
@@ -38,9 +38,14 @@ pub struct Opts {
     #[clap(long, env = "VERBOSE", global = true)]
     pub verbose: bool,
 
-    // Specify the target network: "mainnet" (default), "staging", or NNS URL
+    // Specify the target network: "mainnet" (default), "staging", or a testnet name
     #[clap(long, env = "NETWORK", default_value = "mainnet")]
-    pub network: Network,
+    pub network: String,
+
+    // NNS_URLs for the target network, comma separated.
+    // The argument is mandatory for testnets, and is optional for mainnet and staging
+    #[clap(long, env = "NNS_URLS", aliases = &["registry-url", "nns-url"], value_delimiter = ',')]
+    pub nns_urls: Vec<Url>,
 
     #[clap(subcommand)]
     pub subcommand: Commands,
@@ -409,11 +414,11 @@ pub mod nodes {
 }
 
 #[derive(Clone)]
-pub struct Cli {
-    pub ic_admin: Option<String>,
-    pub nns_url: url::Url,
+pub struct ParsedCli {
+    pub network: Network,
+    pub ic_admin_bin_path: Option<String>,
     pub yes: bool,
-    pub neuron: Option<Neuron>,
+    pub neuron: Neuron,
 }
 
 #[derive(Clone)]
@@ -427,15 +432,11 @@ pub struct UpdateVersion {
     pub versions_to_retire: Option<Vec<String>>,
 }
 
-impl Cli {
-    pub fn get_neuron(&self) -> &Option<Neuron> {
+impl ParsedCli {
+    pub fn get_neuron(&self) -> &Neuron {
         &self.neuron
     }
 
-    pub fn get_nns_url(&self) -> &url::Url {
-        &self.nns_url
-    }
-
     pub fn get_update_cmd_args(update_version: &UpdateVersion) -> Vec<String> {
         [
             [
@@ -462,39 +463,29 @@ impl Cli {
     }
 
     pub async fn from_opts(opts: &Opts, require_authentication: bool) -> anyhow::Result<Self> {
-        let nns_url = opts.network.get_url();
-        let neuron = if let Some(id) = opts.neuron_id {
-            Some(Neuron {
-                id,
-                auth: if let Some(path) = opts.private_key_pem.clone() {
-                    Auth::Keyfile { path }
-                } else if let (Some(slot), Some(pin), Some(key_id)) =
-                    (opts.hsm_slot, opts.hsm_pin.clone(), opts.hsm_key_id.clone())
-                {
-                    Auth::Hsm { pin, slot, key_id }
-                } else {
-                    detect_hsm_auth()?
-                        .ok_or_else(|| anyhow::anyhow!("No valid authentication method found for neuron: {id}"))?
-                },
-            })
-        } else if require_authentication {
-            // Early warn if there will be a problem because a neuron was not detected.
-            match detect_neuron(nns_url.clone()).await {
-                Ok(Some(n)) => Some(n),
-                Ok(None) => {
-                    error!("No neuron detected.  Your HSM device is not detectable (or override variables HSM_PIN, HSM_SLOT, HSM_KEY_ID are incorrectly set); your variables NEURON_ID, PRIVATE_KEY_PEM might not be defined either.");
-                    None
-                },
-                Err(e) => return Err(anyhow::anyhow!("Failed to detect neuron: {}.  Your HSM device is not detectable (or override variables HSM_PIN, HSM_SLOT, HSM_KEY_ID are incorrectly set); your variables NEURON_ID, PRIVATE_KEY_PEM might not be defined either.", e)),
-            }
-        } else {
-            None
-        };
-        Ok(Cli {
+        let network = Network::new(&opts.network, &opts.nns_urls).await.map_err(|e| {
+            anyhow::anyhow!(
+                "Failed to parse network from name {} and NNS urls {:?}. Error: {}",
+                opts.network,
+                opts.nns_urls,
+                e
+            )
+        })?;
+        let neuron = Neuron::new(
+            &network,
+            require_authentication,
+            opts.neuron_id,
+            opts.private_key_pem.clone(),
+            opts.hsm_slot.clone(),
+            opts.hsm_pin.clone(),
+            opts.hsm_key_id.clone(),
+        )
+        .await?;
+        Ok(ParsedCli {
+            network: network,
             yes: opts.yes,
             neuron,
-            ic_admin: opts.ic_admin.clone(),
-            nns_url,
+            ic_admin_bin_path: opts.ic_admin.clone(),
         })
     }
 }
diff --git a/rs/cli/src/clients.rs b/rs/cli/src/clients.rs
@@ -18,7 +18,7 @@ pub struct DashboardBackendClient {
 impl DashboardBackendClient {
     // Only used in tests, which should be cleaned up together with this code.
     #[allow(dead_code)]
-    pub fn new(network: Network, dev: bool) -> DashboardBackendClient {
+    pub fn new(network: &Network, dev: bool) -> DashboardBackendClient {
         Self {
             url: reqwest::Url::parse(if !dev {
                 "https://dashboard.internal.dfinity.network/"
@@ -28,16 +28,12 @@ impl DashboardBackendClient {
             .expect("invalid base url")
             .join("api/proxy/registry/")
             .expect("failed to join url")
-            .join(match network {
-                Network::Mainnet => "mainnet/",
-                Network::Staging => "staging/",
-                Network::Url(_) => "/",
-            })
+            .join(&network.name)
             .expect("failed to join url"),
         }
     }
 
-    pub fn new_with_network_url(url: String) -> Self {
+    pub fn new_with_backend_url(url: String) -> Self {
         Self {
             url: reqwest::Url::parse(&url).unwrap(),
         }
@@ -154,22 +150,28 @@ impl RESTRequestBuilder for reqwest::RequestBuilder {
 mod tests {
     use super::*;
 
-    #[test]
-    fn dashboard_backend_client_url() {
+    #[tokio::test]
+    async fn dashboard_backend_client_url() {
+        let mainnet = Network::new("mainnet", &vec![])
+            .await
+            .expect("failed to create mainnet network");
+        let staging = Network::new("staging", &vec![])
+            .await
+            .expect("failed to create staging network");
         assert_eq!(
-            DashboardBackendClient::new(Network::Mainnet, false).url.to_string(),
+            DashboardBackendClient::new(&mainnet, false).url.to_string(),
             "https://dashboard.internal.dfinity.network/api/proxy/registry/mainnet/"
         );
         assert_eq!(
-            DashboardBackendClient::new(Network::Staging, false).url.to_string(),
+            DashboardBackendClient::new(&staging, false).url.to_string(),
             "https://dashboard.internal.dfinity.network/api/proxy/registry/staging/"
         );
         assert_eq!(
-            DashboardBackendClient::new(Network::Mainnet, true).url.to_string(),
+            DashboardBackendClient::new(&mainnet, true).url.to_string(),
             "http://localhost:17000/api/proxy/registry/mainnet/"
         );
         assert_eq!(
-            DashboardBackendClient::new(Network::Staging, true).url.to_string(),
+            DashboardBackendClient::new(&staging, true).url.to_string(),
             "http://localhost:17000/api/proxy/registry/staging/"
         );
     }