feat(swarm): register templates on startup automatically if not alrea…

…dy registered (#1185)

Description
---
At startup we check for every registered and active templates, then
compare with what we have in templates folder and then we will register
automatically all the missing templates. If any new template in the
meantime added to the folder and not registered, at the next startup
they will be registered automatically as well.
We have a new parameter to be able to pass to `start` command which is
`--disable-template-auto-register` which disables the automatic
registration of templates from the templates directory (by default this
feature is on).

Motivation and Context
---
When we start from a fresh swarm (by deleting `<base_dir>/processes`
folder) all the previously registered templates are gone and not
registered again. We need to have the previously uploaded and registered
templates already in place, so any test or anything can use them.

How Has This Been Tested?
---
### Prepare
  1. Start a swarm locally
2. Upload a compiled template through swarm daemon UI
(http://127.0.0.1:8080/) - I used simply counter example
  3. Stop swarm
  4. delete `<base_dir>/processes` folder (base_dir is mostly `./data`)
5. Make sure that in the `<base_dir>/templates` folder there is the
previously uploaded `.wasm` file (example:
`counter-wasm-e8bd952191282de51ecec1bfdcea1260db3c08bf3ff3b3b072484db4da7fe0c2.wasm`)

### Test
  1. Start a swarm locally again
  2. Wait for it to register all nodes and start consensus
3. Check any of the Validator Node's UI templates list if we have the
new templates. (example URL: `http://localhost:12072/templates`)

Also tested the scenarios when I skip registration by passing the param
above and checked also when I start multiple times without touching data
directory whether the template is registered multiple times (it's not).

What process can a PR reviewer use to test or verify this change?
---
See test.

Breaking Changes
---

- [x] None
- [ ] Requires data directory to be deleted
- [ ] Other - Please specify

applications/tari_swarm_daemon/src/cli.rs

@@ -71,6 +71,8 @@ pub struct Overrides {
     pub start_port: Option<u16>,
     #[clap(short = 'k', long)]
     pub skip_registration: bool,
+    #[clap(long)]
+    pub disable_template_auto_register: bool,
 }
 
 impl Overrides {
@@ -105,6 +107,10 @@ impl Overrides {
             config.start_port = port;
         }
 
+        if self.disable_template_auto_register {
+            config.auto_register_previous_templates = false;
+        }
+
         Ok(())
     }
 }

applications/tari_swarm_daemon/src/config.rs

@@ -28,6 +28,12 @@ pub struct Config {
     pub processes: ProcessesConfig,
     #[serde(default)]
     pub skip_registration: bool,
+    #[serde(default = "default_as_true")]
+    pub auto_register_previous_templates: bool,
+}
+
+fn default_as_true() -> bool {
+    true
 }
 
 impl Config {

applications/tari_swarm_daemon/src/main.rs

@@ -185,6 +185,7 @@ fn get_base_config(cli: &Cli) -> anyhow::Result<Config> {
             instances,
             executables,
         },
+        auto_register_previous_templates: true,
     })
 }
 

applications/tari_swarm_daemon/src/process_manager/handle.rs

@@ -61,6 +61,7 @@ pub enum ProcessManagerRequest {
     },
 }
 
+#[derive(Debug)]
 pub struct TemplateData {
     pub name: String,
     pub version: u32,

applications/tari_swarm_daemon/src/process_manager/manager.rs

@@ -1,15 +1,19 @@
 //   Copyright 2024 The Tari Project
 //   SPDX-License-Identifier: BSD-3-Clause
 
-use std::{collections::HashMap, fs::File, path::PathBuf, time::Duration};
+use std::{collections::HashMap, fs::File, path::PathBuf, str::FromStr, time::Duration};
 
 use anyhow::{anyhow, Context};
 use log::info;
 use minotari_node_grpc_client::grpc;
+use tari_common_types::types::FixedHash;
 use tari_crypto::tari_utilities::ByteArray;
-use tari_engine_types::TemplateAddress;
+use tari_dan_engine::wasm::WasmModule;
+use tari_engine_types::{calculate_template_binary_hash, TemplateAddress};
 use tari_shutdown::ShutdownSignal;
+use tari_validator_node_client::types::GetTemplatesRequest;
 use tokio::{sync::mpsc, time::sleep};
+use url::Url;
 
 use crate::{
     config::{Config, InstanceType},
@@ -28,6 +32,9 @@ pub struct ProcessManager {
     rx_request: mpsc::Receiver<ProcessManagerRequest>,
     shutdown_signal: ShutdownSignal,
     skip_registration: bool,
+    disable_template_auto_register: bool,
+    base_dir: PathBuf,
+    web_server_port: u16,
 }
 
 impl ProcessManager {
@@ -47,6 +54,9 @@ impl ProcessManager {
             ),
             rx_request,
             shutdown_signal,
+            disable_template_auto_register: !config.auto_register_previous_templates,
+            base_dir: config.base_dir.clone(),
+            web_server_port: config.webserver.bind_address.port(),
         };
         (this, ProcessManagerHandle::new(tx_request))
     }
@@ -73,9 +83,99 @@ impl ProcessManager {
                 .context("registering validator node via GRPC")?;
         }
 
+        if !self.disable_template_auto_register {
+            let registered_templates = self.registered_templates().await?;
+            let registered_template_names: Vec<String> = registered_templates
+                .iter()
+                .map(|template_data| format!("{}-{}", template_data.name, template_data.version))
+                .collect();
+            let fs_templates = self.file_system_templates().await?;
+            for template_data in fs_templates.iter().filter(|fs_template_data| {
+                !registered_template_names.contains(&format!("{}-{}", fs_template_data.name, fs_template_data.version))
+            }) {
+                info!(
+                    "🟡 Register missing template from local file system: {}",
+                    template_data.name
+                );
+                self.register_template(TemplateData {
+                    name: template_data.name.clone(),
+                    version: template_data.version,
+                    contents_hash: template_data.contents_hash,
+                    contents_url: template_data.contents_url.clone(),
+                })
+                .await?;
+            }
+        }
+
         Ok(())
     }
 
+    /// Loads all the file system templates from the standard `<BASE_DIR>/templates` dir.
+    async fn file_system_templates(&self) -> anyhow::Result<Vec<TemplateData>> {
+        let templates_dir = self.base_dir.join("templates");
+        let mut templates_dir_content = tokio::fs::read_dir(templates_dir).await?;
+        let mut result = vec![];
+        while let Some(dir_entry) = templates_dir_content.next_entry().await? {
+            if dir_entry.path().is_file() {
+                if let Some(extension) = dir_entry.path().extension() {
+                    if extension == "wasm" {
+                        let file_name = dir_entry.file_name();
+                        let file_name = file_name.to_str().ok_or(anyhow!("Can't get file name!"))?;
+                        let file_content = tokio::fs::read(dir_entry.path()).await?;
+                        let loaded = WasmModule::load_template_from_code(file_content.as_slice())?;
+                        let name = loaded.template_def().template_name().to_string();
+                        let hash = calculate_template_binary_hash(&file_content);
+                        result.push(TemplateData {
+                            name,
+                            version: 0,
+                            contents_hash: hash,
+                            contents_url: Url::parse(&format!(
+                                "http://localhost:{}/templates/{}",
+                                self.web_server_port, file_name
+                            ))?,
+                        })
+                    }
+                }
+            }
+        }
+
+        Ok(result)
+    }
+
+    /// Loads all already registered templates.
+    async fn registered_templates(&self) -> anyhow::Result<Vec<TemplateData>> {
+        let process = self.instance_manager.validator_nodes().next().ok_or_else(|| {
+            anyhow!(
+                "No MinoTariConsoleWallet instances found. Please start a wallet before trying to get active templates"
+            )
+        })?;
+
+        let mut client = process.connect_client()?;
+        Ok(client
+            .get_active_templates(GetTemplatesRequest { limit: 10_000 })
+            .await?
+            .templates
+            .iter()
+            .map(|metadata| {
+                let url = if let Ok(url) = Url::from_str(metadata.url.as_str()) {
+                    url
+                } else {
+                    Url::parse(&format!(
+                        "http://localhost:{}/templates/{}",
+                        self.web_server_port, metadata.name
+                    ))
+                    .unwrap()
+                };
+                TemplateData {
+                    name: metadata.name.clone(),
+                    version: 0,
+                    contents_hash: FixedHash::try_from(metadata.binary_sha.as_slice()).unwrap_or_default(),
+                    contents_url: url,
+                }
+            })
+            .collect())
+    }
+
     fn check_instances_running(&mut self) -> anyhow::Result<()> {
         for instance in self
             .instance_manager