update monitoring script

ava-labs · Nov 15, 2023 · 1f3e017 · 1f3e017
1 parent 4a24823
commit 1f3e017
Show file tree

Hide file tree

Showing 6 changed files with 500 additions and 25 deletions.
diff --git a/cmd/nodecmd/create.go b/cmd/nodecmd/create.go
@@ -386,7 +386,10 @@ func runAnsible(inventoryPath string, network models.Network, avalancheGoVersion
 	); err != nil {
 		return err
 	}
-	return ansible.RunAnsiblePlaybookSetupMonitoring(app.GetAnsibleDir(), inventoryPath, ansibleHostIDs)
+	if !separateMonitoringInstance {
+		return ansible.RunAnsiblePlaybookSetupMonitoring(app.GetAnsibleDir(), inventoryPath, ansibleHostIDs)
+	}
+	return ansible.RunAnsiblePlaybookSetupSeparateMonitoring(app.GetAnsibleDir(), inventoryPath, ansibleHostIDs, app.GetMonitoringScriptFile(), "\\'localhost:9650\\',\\'54.215.90.166:9650\\'", "\\'localhost:9100\\',\\'54.215.90.166:9100\\'")
 }
 
 func setupBuildEnv(inventoryPath, ansibleHostIDs string) error {

diff --git a/pkg/ansible/ansible.go b/pkg/ansible/ansible.go
@@ -576,3 +576,21 @@ func RunAnsiblePlaybookSetupMonitoring(ansibleDir, inventoryPath, ansibleHostIDs
 	}
 	return cmdErr
 }
+
+// RunAnsiblePlaybookSetupSeparateMonitoring sets up monitoring in a separate cloud server
+// targets all hosts in ansible inventory file
+func RunAnsiblePlaybookSetupSeparateMonitoring(ansibleDir, inventoryPath, ansibleHostIDs, monitoringScriptPath, arg1, arg2 string) error {
+	fmt.Printf("running RunAnsiblePlaybookSetupSeparateMonitoring %s %s %s \n", monitoringScriptPath, arg1, arg2)
+	playbookInputs := "target=" + ansibleHostIDs + " monitoringScriptPath=" + monitoringScriptPath + " arg1=" + arg1 + " arg2=" + arg2
+	cmd := exec.Command(constants.AnsiblePlaybook, constants.SetupNodeSeparateMonitoringPlaybook, constants.AnsibleInventoryFlag, inventoryPath, constants.AnsibleExtraVarsFlag, playbookInputs, constants.AnsibleExtraArgsIdentitiesOnlyFlag) //nolint:gosec
+	cmd.Dir = ansibleDir
+	stdoutBuffer, stderrBuffer := utils.SetupRealtimeCLIOutput(cmd, true, true)
+	cmdErr := cmd.Run()
+	if err := displayErrMsg(stdoutBuffer); err != nil {
+		return err
+	}
+	if err := displayErrMsg(stderrBuffer); err != nil {
+		return err
+	}
+	return cmdErr
+}
diff --git a/pkg/ansible/playbook/setupSeparateMonitoring.yml b/pkg/ansible/playbook/setupSeparateMonitoring.yml
@@ -0,0 +1,18 @@
+---
+- hosts: "{{ target }}"
+  gather_facts: no
+  tasks:
+    - name: copy monitoring script to cloud server
+      copy:
+        src: "{{ monitoringScriptPath }}"
+        dest: /home/ubuntu/
+    - name: modify permission for monitoring script
+      shell: chmod 755 monitoring-separate-installer.sh
+    - name: set up Prometheus
+      shell: ./monitoring-separate-installer.sh --1
+    - name: install Grafana
+      shell: ./monitoring-separate-installer.sh --2
+#    - name: set up node_exporter
+#      shell: ./monitoring-separate-installer.sh --3 {{ arg1 }} {{ arg2 }}
+#    - name: set up dashboards
+#      shell: ./monitoring-separate-installer.sh --4
diff --git a/pkg/application/app.go b/pkg/application/app.go
@@ -132,6 +132,10 @@ func (app *Avalanche) GetAnsibleDir() string {
 	return filepath.Join(app.GetNodesDir(), constants.AnsibleDir)
 }
 
+func (app *Avalanche) GetMonitoringDir() string {
+	return filepath.Join(app.GetNodesDir(), constants.MonitoringDir)
+}
+
 func (app *Avalanche) CreateAnsibleDir() error {
 	ansibleDir := app.GetAnsibleDir()
 	if _, err := os.Stat(ansibleDir); os.IsNotExist(err) {
@@ -739,6 +743,10 @@ func (app *Avalanche) GetBootstrappedJSONFile() string {
 	return filepath.Join(app.GetAnsibleStatusDir(), constants.IsBootstrappedJSONFile)
 }
 
+func (app *Avalanche) GetMonitoringScriptFile() string {
+	return filepath.Join(app.GetMonitoringDir(), constants.MonitoringScriptFile)
+}
+
 func (app *Avalanche) GetHealthyJSONFile() string {
 	return filepath.Join(app.GetAnsibleStatusDir(), constants.IsHealthyJSONFile)
 }

diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
@@ -29,6 +29,7 @@ const (
 	NodeCloudConfigFileName      = "node_cloud_config.json"
 	TerraformDir                 = "terraform"
 	AnsibleDir                   = "ansible"
+	MonitoringDir                = "monitoring"
 	AnsibleHostInventoryFileName = "hosts"
 	StopAWSNode                  = "stop-aws-node"
 	CreateAWSNode                = "create-aws-node"
@@ -133,6 +134,7 @@ const (
 	BuildEnvGolangVersion                        = "1.21.1"
 	IsHealthyJSONFile                            = "isHealthy.json"
 	IsBootstrappedJSONFile                       = "isBootstrapped.json"
+	MonitoringScriptFile                         = "monitoring-separate-installer.sh"
 	AvalancheGoVersionJSONFile                   = "avalancheGoVersion.json"
 	SubnetSyncJSONFile                           = "isSubnetSynced.json"
 	AnsibleInventoryDir                          = "inventories"
@@ -153,30 +155,33 @@ const (
 	OldMetricsConfigFileName     = ".avalanche-cli/config"
 	DefaultConfigFileName        = ".avalanche-cli/config.json"
 
-	AWSCloudService             = "Amazon Web Services"
-	GCPCloudService             = "Google Cloud Platform"
-	AnsibleSSHUser              = "ubuntu"
-	AWSNodeAnsiblePrefix        = "aws_node"
-	GCPNodeAnsiblePrefix        = "gcp_node"
-	CustomVMDir                 = "vms"
-	GCPStaticIPPrefix           = "static-ip"
-	AvaLabsOrg                  = "ava-labs"
-	AvalancheGoRepoName         = "avalanchego"
-	SubnetEVMRepoName           = "subnet-evm"
-	CliRepoName                 = "avalanche-cli"
-	UpgradeAvalancheGoPlaybook  = "playbook/upgradeAvalancheGo.yml"
-	UpgradeSubnetEVMPlaybook    = "playbook/upgradeSubnetEVM.yml"
-	StopNodePlaybook            = "playbook/stopNode.yml"
-	StartNodePlaybook           = "playbook/startNode.yml"
-	GetNewSubnetEVMPlaybook     = "playbook/getNewSubnetEVMRelease.yml"
-	SubnetEVMReleaseURL         = "https://github.com/ava-labs/subnet-evm/releases/download/%s/%s"
-	SubnetEVMArchive            = "subnet-evm_%s_linux_amd64.tar.gz"
-	SubnetEVMBinaryPath         = "/home/ubuntu/.avalanchego/plugins/%s"
-	AvalanchegoMonitoringPort   = 9090
-	AvalanchegoGrafanaPort      = 3000
-	SetupNodeMonitoringPlaybook = "playbook/setupMonitoring.yml"
-	AvalancheGoInstallDir       = "avalanchego"
-	SubnetEVMInstallDir         = "subnet-evm"
+	AWSCloudService                     = "Amazon Web Services"
+	GCPCloudService                     = "Google Cloud Platform"
+	AnsibleSSHUser                      = "ubuntu"
+	AWSNodeAnsiblePrefix                = "aws_node"
+	GCPNodeAnsiblePrefix                = "gcp_node"
+	CustomVMDir                         = "vms"
+	GCPStaticIPPrefix                   = "static-ip"
+	AvaLabsOrg                          = "ava-labs"
+	AvalancheGoRepoName                 = "avalanchego"
+	SubnetEVMRepoName                   = "subnet-evm"
+	CliRepoName                         = "avalanche-cli"
+	UpgradeAvalancheGoPlaybook          = "playbook/upgradeAvalancheGo.yml"
+	UpgradeSubnetEVMPlaybook            = "playbook/upgradeSubnetEVM.yml"
+	StopNodePlaybook                    = "playbook/stopNode.yml"
+	StartNodePlaybook                   = "playbook/startNode.yml"
+	GetNewSubnetEVMPlaybook             = "playbook/getNewSubnetEVMRelease.yml"
+	SubnetEVMReleaseURL                 = "https://github.com/ava-labs/subnet-evm/releases/download/%s/%s"
+	SubnetEVMArchive                    = "subnet-evm_%s_linux_amd64.tar.gz"
+	SubnetEVMBinaryPath                 = "/home/ubuntu/.avalanchego/plugins/%s"
+	AvalanchegoMonitoringPort           = 9090
+	AvalanchegoGrafanaPort              = 3000
+	MetricsLocalHost                    = "localhost:9090"
+	AvalancheGoLocalHost                = "localhost:9100"
+	SetupNodeMonitoringPlaybook         = "playbook/setupMonitoring.yml"
+	SetupNodeSeparateMonitoringPlaybook = "playbook/setupSeparateMonitoring.yml"
+	AvalancheGoInstallDir               = "avalanchego"
+	SubnetEVMInstallDir                 = "subnet-evm"
 
 	SubnetEVMBin = "subnet-evm"