feat(watcher): add mattermost notifier for channel alerting (#1126)

Description
---
Adds alerting for a channel on Mattermost with a generalized notifier
implementation. Next step is to add Telegram support.

How Has This Been Tested?
---
Create a Mattermost channel and log in to the server using the api to
obtain credentials. After updating the config file (with both
`channel_id` and `credentials`), run `tari_swarm_daemon` together with
the `tari_watcher`, and go through the initial registration step,
warning on close to expiry, and re-registration. Observe messages posted
in the channel.

applications/tari_watcher/Cargo.toml

@@ -34,6 +34,8 @@ log = { workspace = true }
 fern = { workspace = true, features = ["colored"] }
 tonic = { workspace = true }
 json5 = { workspace = true }
+reqwest = { workspace = true, features = ["json", "blocking", "rustls-tls"] }
+serde_json = { workspace = true }
 
 toml = "0.8.12"
 humantime = "2.1.0"

applications/tari_watcher/README.md

@@ -0,0 +1,43 @@
+# Tari Watcher
+
+**Features**:
+* Registers the validator node on L2 by sending a transaction on L1
+* Monitors the chain and warns when registration is near expiration
+* Automatically re-registers the node
+* Alerts on Mattermost and Telegram
+
+### Quickstart
+
+Initialize the project with `tari_watcher init` and start it with `tari_watcher run`. Edit the newly generated `config.toml` to enable notifications on channels such as Mattermost and Telegram.Make sure to have started up `tari_validator_node` once previously to have a node directory set up, default is `tari_validator_node -- -b data/vn1`.
+
+### Setup
+
+The default values used (see `constants.rs`) when running the project without any flags:
+```
+- DEFAULT_MAIN_PROJECT_PATH: base directory, the same level as the repository `tari-dan`
+- DEFAULT_WATCHER_CONFIG_PATH: relative to the base directory, main configuration file
+- DEFAULT_VALIDATOR_KEY_PATH: relative to the base directory, validator node registration file
+- DEFAULT_VALIDATOR_NODE_BINARY_PATH: relative to the base directory, default is Rust build directory `target/release`
+- DEFAULT_VALIDATOR_DIR: relative to the project base directory, home directory for everything validator node
+- DEFAULT_MINOTARI_MINER_BINARY_PATH: relative to the base directory, default is Rust build directory `target/release`
+- DEFAULT_BASE_NODE_GRPC_ADDRESS: default is Tari swarm localhost and port
+- DEFAULT_BASE_WALLET_GRPC_ADDRESS: default is Tari swarm localhost and port
+```
+
+### Project
+
+```
+├── alerting.rs     # channel notifier implementations
+├── cli.rs          # cli and flags passed during bootup
+├── config.rs       # main config file creation 
+├── constants.rs    # various constants used as default values
+├── helpers.rs      # common helper functions
+├── logger.rs
+├── main.rs
+├── manager.rs      # manages the spawn validator node process and receives requests
+├── minotari.rs     # communicates with the base node (L1)
+├── monitoring.rs   # outputs logs and sends the alerts
+├── process.rs      # spawns the validator node process and sets up the channels
+├── registration.rs # handles the logic for sending a node registration transaction
+└── shutdown.rs
+```

applications/tari_watcher/src/alerting.rs

@@ -0,0 +1,97 @@
+// Copyright 2024 The Tari Project
+// SPDX-License-Identifier: BSD-3-Clause
+
+use anyhow::{bail, Result};
+use reqwest::StatusCode;
+use serde_json::json;
+
+pub trait Alerting {
+    fn new(url: String, channel_id: String, credentials: String) -> Self;
+
+    // Sends an alert message to the service
+    async fn alert(&mut self, message: &str) -> Result<()>;
+
+    // Checks that the service is reachable
+    async fn ping(&self) -> Result<()>;
+
+    // Statistics on the alerts sent
+    // todo: expand granularity and types of stats
+    fn stats(&self) -> Result<u64>;
+}
+
+pub struct MatterMostNotifier {
+    // Mattermost server URL
+    server_url: String,
+    // Mattermost channel ID used for alerts
+    channel_id: String,
+    // User token (retrieved after login)
+    credentials: String,
+    // Alerts sent since last reset
+    alerts_sent: u64,
+    // HTTP client
+    client: reqwest::Client,
+}
+
+impl Alerting for MatterMostNotifier {
+    fn new(server_url: String, channel_id: String, credentials: String) -> Self {
+        Self {
+            server_url,
+            channel_id,
+            credentials,
+            alerts_sent: 0,
+            client: reqwest::Client::new(),
+        }
+    }
+
+    async fn alert(&mut self, message: &str) -> Result<()> {
+        const LOGIN_ENDPOINT: &str = "/api/v4/posts";
+        let url = format!("{}{}", self.server_url, LOGIN_ENDPOINT);
+        let req = json!({
+            "channel_id": self.channel_id,
+            "message": message,
+        });
+        let resp = self
+            .client
+            .post(&url)
+            .json(&req)
+            .header("Authorization", format!("Bearer {}", self.credentials))
+            .send()
+            .await?;
+
+        if resp.status() != StatusCode::CREATED {
+            bail!("Failed to send alert, got response: {}", resp.status());
+        }
+
+        self.alerts_sent += 1;
+
+        Ok(())
+    }
+
+    async fn ping(&self) -> Result<()> {
+        const PING_ENDPOINT: &str = "/api/v4/users/me";
+        if self.server_url.is_empty() {
+            bail!("Server URL is empty");
+        }
+        if self.credentials.is_empty() {
+            bail!("Credentials are empty");
+        }
+
+        let url = format!("{}{}", self.server_url, PING_ENDPOINT);
+        let resp = self
+            .client
+            .get(url.clone())
+            .header("Authorization", format!("Bearer {}", self.credentials))
+            .send()
+            .await?;
+
+        if resp.status() != StatusCode::OK {
+            bail!("Failed to ping, got response: {}", resp.status());
+        }
+
+        Ok(())
+    }
+
+    fn stats(&self) -> Result<u64> {
+        Ok(self.alerts_sent)
+    }
+}

applications/tari_watcher/src/cli.rs

@@ -7,7 +7,12 @@ use clap::Parser;
 
 use crate::{
     config::{Config, InstanceType},
-    constants::{DEFAULT_PROJECT_ROOT, DEFAULT_WATCHER_CONFIG_PATH},
+    constants::{
+        DEFAULT_MAIN_PROJECT_PATH,
+        DEFAULT_VALIDATOR_DIR,
+        DEFAULT_VALIDATOR_KEY_PATH,
+        DEFAULT_WATCHER_CONFIG_PATH,
+    },
 };
 
 #[derive(Clone, Debug, Parser)]
@@ -30,10 +35,14 @@ impl Cli {
 
 #[derive(Debug, Clone, clap::Args)]
 pub struct CommonCli {
-    #[clap(short = 'b', long, parse(from_os_str), default_value = DEFAULT_PROJECT_ROOT)]
+    #[clap(short = 'b', long, parse(from_os_str), default_value = DEFAULT_MAIN_PROJECT_PATH)]
     pub base_dir: PathBuf,
     #[clap(short = 'c', long, parse(from_os_str), default_value = DEFAULT_WATCHER_CONFIG_PATH)]
     pub config_path: PathBuf,
+    #[clap(short = 'k', long, parse(from_os_str), default_value = DEFAULT_VALIDATOR_KEY_PATH)]
+    pub key_path: PathBuf,
+    #[clap(short = 'v', long, parse(from_os_str), default_value = DEFAULT_VALIDATOR_DIR)]
+    pub validator_dir: PathBuf,
 }
 
 #[derive(Clone, Debug, clap::Subcommand)]