diff --git a/cmd/nodecmd/create.go b/cmd/nodecmd/create.go index 52e5da7e7..6be4f5061 100644 --- a/cmd/nodecmd/create.go +++ b/cmd/nodecmd/create.go @@ -6,31 +6,31 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" + "net/http" "os" - "os/exec" "os/user" "path/filepath" - "strings" - "time" + "sync" "github.com/ava-labs/avalanche-cli/cmd/subnetcmd" "github.com/ava-labs/avalanche-cli/pkg/ansible" awsAPI "github.com/ava-labs/avalanche-cli/pkg/aws" gcpAPI "github.com/ava-labs/avalanche-cli/pkg/gcp" + "github.com/ava-labs/avalanche-cli/pkg/ssh" "github.com/ava-labs/avalanche-cli/pkg/terraform" "github.com/ava-labs/avalanche-cli/pkg/utils" "github.com/ava-labs/avalanche-cli/pkg/vm" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/staking" + "golang.org/x/exp/slices" "github.com/ava-labs/avalanche-cli/pkg/constants" "github.com/ava-labs/avalanche-cli/pkg/models" "github.com/ava-labs/avalanche-cli/pkg/ux" "github.com/spf13/cobra" - - "golang.org/x/sync/errgroup" ) const ( @@ -217,31 +217,77 @@ func createNodes(_ *cobra.Command, args []string) error { if err != nil { return err } - - time.Sleep(30 * time.Second) - + inventoryPath := app.GetAnsibleInventoryDirPath(clusterName) avalancheGoVersion, err := getAvalancheGoVersion() if err != nil { return err } - inventoryPath := app.GetAnsibleInventoryDirPath(clusterName) if err = ansible.CreateAnsibleHostInventory(inventoryPath, cloudConfig.CertFilePath, cloudService, publicIPMap); err != nil { return err } + if err := updateAnsiblePublicIPs(clusterName); err != nil { + return err + } + allHosts, err := ansible.GetInventoryFromAnsibleInventoryFile(inventoryPath) + if err != nil { + return err + } + hosts := utils.Filter(allHosts, func(h models.Host) bool { return slices.Contains(cloudConfig.InstanceIDs, h.GetCloudID()) }) + // waiting for all nodes to become accessible + failedHosts := waitForHosts(hosts) + if failedHosts.Len() > 0 { + for _, result := range failedHosts.GetResults() { + ux.Logger.PrintToUser("Instance %s failed to provision with error %s. Please check instance logs for more information", result.NodeID, result.Err) + } + return fmt.Errorf("failed to provision node(s) %s", failedHosts.GetNodeList()) + } - ux.Logger.PrintToUser("Installing AvalancheGo and Avalanche-CLI and starting bootstrap process on the newly created Avalanche node(s) ...") ansibleHostIDs, err := utils.MapWithError(cloudConfig.InstanceIDs, func(s string) (string, error) { return models.HostCloudIDToAnsibleID(cloudService, s) }) if err != nil { return err } - createdAnsibleHostIDs := strings.Join(ansibleHostIDs, ",") - if err = runAnsible(inventoryPath, network, avalancheGoVersion, clusterName, createdAnsibleHostIDs); err != nil { - return err + ux.Logger.PrintToUser("Installing AvalancheGo and Avalanche-CLI and starting bootstrap process on the newly created Avalanche node(s) ...") + wg := sync.WaitGroup{} + wgResults := models.NodeResults{} + for _, host := range hosts { + wg.Add(1) + go func(nodeResults *models.NodeResults, host models.Host) { + defer wg.Done() + if err := host.Connect(constants.SSHScriptTimeout); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + defer func() { + if err := host.Disconnect(); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + } + }() + if err := provideStakingCertAndKey(host); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + if err := ssh.RunSSHSetupNode(host, app.Conf.GetConfigPath(), avalancheGoVersion, network.Kind == models.Devnet); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + if err := ssh.RunSSHSetupBuildEnv(host); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + }(&wgResults, host) } - if err = setupBuildEnv(inventoryPath, createdAnsibleHostIDs); err != nil { - return err + wg.Wait() + ux.Logger.PrintToUser("======================================") + ux.Logger.PrintToUser("AVALANCHE NODE(S) STATUS") + ux.Logger.PrintToUser("======================================") + ux.Logger.PrintToUser("") + for _, node := range hosts { + if wgResults.HasNodeIDWithError(node.NodeID) { + ux.Logger.PrintToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErroHostMap()[node.NodeID]) + } else { + ux.Logger.PrintToUser("Node %s is CREATED", node.NodeID) + } } - if network.Kind == models.Devnet { ux.Logger.PrintToUser("Setting up Devnet ...") if err := setupDevnet(clusterName); err != nil { @@ -249,8 +295,12 @@ func createNodes(_ *cobra.Command, args []string) error { } } - printResults(cloudConfig, publicIPMap, ansibleHostIDs) - ux.Logger.PrintToUser("AvalancheGo and Avalanche-CLI installed and node(s) are bootstrapping!") + if wgResults.HasErrors() { + return fmt.Errorf("failed to deploy node(s) %s", wgResults.GetErroHosts()) + } else { + printResults(cloudConfig, publicIPMap, ansibleHostIDs) + ux.Logger.PrintToUser("AvalancheGo and Avalanche-CLI installed and node(s) are bootstrapping!") + } return nil } @@ -341,23 +391,6 @@ func setupAnsible(clusterName string) error { return updateAnsiblePublicIPs(clusterName) } -func runAnsible(inventoryPath string, network models.Network, avalancheGoVersion, clusterName, ansibleHostIDs string) error { - if err := setupAnsible(clusterName); err != nil { - return err - } - if err := distributeStakingCertAndKey(strings.Split(ansibleHostIDs, ","), inventoryPath); err != nil { - return err - } - return ansible.RunAnsiblePlaybookSetupNode( - app.Conf.GetConfigPath(), - app.GetAnsibleDir(), - inventoryPath, - avalancheGoVersion, - fmt.Sprint(network.Kind == models.Devnet), - ansibleHostIDs, - ) -} - func setupBuildEnv(inventoryPath, ansibleHostIDs string) error { ux.Logger.PrintToUser("Installing Custom VM build environment on the cloud server(s) ...") ansibleTargetHosts := "all" @@ -417,46 +450,44 @@ func generateNodeCertAndKeys(stakerCertFilePath, stakerKeyFilePath, blsKeyFilePa return nodeID, nil } -func distributeStakingCertAndKey(ansibleHostIDs []string, inventoryPath string) error { - ux.Logger.PrintToUser("Generating staking keys in local machine...") - eg := errgroup.Group{} - for _, ansibleInstanceID := range ansibleHostIDs { - _, instanceID, err := models.HostAnsibleIDToCloudID(ansibleInstanceID) - if err != nil { - return err - } - keyPath := filepath.Join(app.GetNodesDir(), instanceID) - eg.Go(func() error { - nodeID, err := generateNodeCertAndKeys( - filepath.Join(keyPath, constants.StakerCertFileName), - filepath.Join(keyPath, constants.StakerKeyFileName), - filepath.Join(keyPath, constants.BLSKeyFileName), - ) - if err != nil { - ux.Logger.PrintToUser("Failed to generate staking keys for host %s", instanceID) - return err - } else { - ux.Logger.PrintToUser("Generated staking keys for host %s[%s] ", instanceID, nodeID.String()) - } - return nil - }) - } - if err := eg.Wait(); err != nil { +func provideStakingCertAndKey(host models.Host) error { + instanceID := host.GetCloudID() + keyPath := filepath.Join(app.GetNodesDir(), instanceID) + nodeID, err := generateNodeCertAndKeys( + filepath.Join(keyPath, constants.StakerCertFileName), + filepath.Join(keyPath, constants.StakerKeyFileName), + filepath.Join(keyPath, constants.BLSKeyFileName), + ) + if err != nil { + ux.Logger.PrintToUser("Failed to generate staking keys for host %s", instanceID) return err + } else { + ux.Logger.PrintToUser("Generated staking keys for host %s[%s] ", instanceID, nodeID.String()) } - ux.Logger.PrintToUser("Copying staking keys to remote machine(s)...") - return ansible.RunAnsiblePlaybookCopyStakingFiles(app.GetAnsibleDir(), strings.Join(ansibleHostIDs, ","), app.GetNodesDir(), inventoryPath) + return ssh.RunSSHUploadStakingFiles(host, keyPath) } func getIPAddress() (string, error) { - ipOutput, err := exec.Command("curl", "https://api.ipify.org?format=json").Output() + resp, err := http.Get("https://api.ipify.org?format=json") + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.New("HTTP request failed") + } + + body, err := io.ReadAll(resp.Body) if err != nil { return "", err } + var result map[string]interface{} - if err = json.Unmarshal(ipOutput, &result); err != nil { + if err := json.Unmarshal(body, &result); err != nil { return "", err } + ipAddress, ok := result["ip"].(string) if ok { if net.ParseIP(ipAddress) == nil { @@ -464,6 +495,7 @@ func getIPAddress() (string, error) { } return ipAddress, nil } + return "", errors.New("no IP address found") } @@ -592,3 +624,21 @@ func printResults(cloudConfig CloudConfig, publicIPMap map[string]string, ansibl ux.Logger.PrintToUser(fmt.Sprintf("Don't delete or replace your ssh private key file at %s as you won't be able to access your cloud server without it", cloudConfig.CertFilePath)) ux.Logger.PrintToUser("") } + +// waitForHosts waits for all hosts to become available via SSH. +func waitForHosts(hosts []models.Host) *models.NodeResults { + hostErrors := models.NodeResults{} + createdWaitGroup := sync.WaitGroup{} + for _, host := range hosts { + createdWaitGroup.Add(1) + go func(nodeResults *models.NodeResults, host models.Host) { + defer createdWaitGroup.Done() + if err := host.WaitForSSHShell(2 * constants.SSHFileOpsTimeout); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + }(&hostErrors, host) + } + createdWaitGroup.Wait() + return &hostErrors +} diff --git a/cmd/nodecmd/create_devnet.go b/cmd/nodecmd/create_devnet.go index c3484fbfe..6f7107209 100644 --- a/cmd/nodecmd/create_devnet.go +++ b/cmd/nodecmd/create_devnet.go @@ -9,12 +9,14 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/ava-labs/avalanche-cli/pkg/ansible" "github.com/ava-labs/avalanche-cli/pkg/constants" "github.com/ava-labs/avalanche-cli/pkg/key" "github.com/ava-labs/avalanche-cli/pkg/models" + "github.com/ava-labs/avalanche-cli/pkg/ssh" "github.com/ava-labs/avalanche-cli/pkg/utils" "github.com/ava-labs/avalanche-cli/pkg/ux" "github.com/ava-labs/avalanchego/config" @@ -111,10 +113,8 @@ func setupDevnet(clusterName string) error { if err := checkCluster(clusterName); err != nil { return err } - if err := setupAnsible(clusterName); err != nil { - return err - } - ansibleHostIDs, err := ansible.GetAnsibleHostsFromInventory(app.GetAnsibleInventoryDirPath(clusterName)) + inventoryPath := app.GetAnsibleInventoryDirPath(clusterName) + ansibleHostIDs, err := ansible.GetAnsibleHostsFromInventory(inventoryPath) if err != nil { return err } @@ -188,16 +188,45 @@ func setupDevnet(clusterName string) error { return err } } - // update node/s genesis + conf and start - if err := ansible.RunAnsiblePlaybookSetupDevnet( - app.GetAnsibleDir(), - strings.Join(ansibleHostIDs, ","), - app.GetNodesDir(), - app.GetAnsibleInventoryDirPath(clusterName), - ); err != nil { + hosts, err := ansible.GetInventoryFromAnsibleInventoryFile(inventoryPath) + if err != nil { return err } + wg := sync.WaitGroup{} + wgResults := models.NodeResults{} + for _, host := range hosts { + wg.Add(1) + go func(nodeResults *models.NodeResults, host models.Host) { + defer wg.Done() + if err := host.Connect(constants.SSHScriptTimeout); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + defer func() { + if err := host.Disconnect(); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + } + }() + keyPath := filepath.Join(app.GetNodesDir(), host.GetCloudID()) + if err := ssh.RunSSHSetupDevNet(host, keyPath); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + return + } + }(&wgResults, host) + } + wg.Wait() + for _, node := range hosts { + if wgResults.HasNodeIDWithError(node.NodeID) { + ux.Logger.PrintToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErroHostMap()[node.NodeID]) + } else { + ux.Logger.PrintToUser("Node %s is SETUP as devnet", node.NodeID) + } + } + // stop execution if at least one node failed + if wgResults.HasErrors() { + return fmt.Errorf("failed to deploy node(s) %s", wgResults.GetErroHosts()) + } // update cluster config with network information clustersConfig, err := app.LoadClustersConfig() diff --git a/cmd/nodecmd/upgrade.go b/cmd/nodecmd/upgrade.go index adb201e2c..8ebdca12e 100644 --- a/cmd/nodecmd/upgrade.go +++ b/cmd/nodecmd/upgrade.go @@ -72,7 +72,7 @@ func upgrade(_ *cobra.Command, args []string) error { return err } for _, vmID := range upgradeInfo.SubnetEVMIDsToUpgrade { - subnetEVMBinaryPath := fmt.Sprintf(constants.SubnetEVMBinaryPath, vmID) + subnetEVMBinaryPath := fmt.Sprintf(constants.CloudNodeSubnetEvmBinaryPath, vmID) if err = upgradeSubnetEVM(clusterName, subnetEVMBinaryPath, node, upgradeInfo.SubnetEVMVersion); err != nil { return err } diff --git a/go.mod b/go.mod index 5674aa8db..db8c04954 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/hashicorp/hcl/v2 v2.17.0 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/manifoldco/promptui v0.9.0 + github.com/melbahja/goph v1.4.0 github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.10 @@ -26,6 +27,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/zclconf/go-cty v1.13.0 go.uber.org/zap v1.24.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/mod v0.12.0 golang.org/x/net v0.17.0 @@ -115,6 +117,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.15.15 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -132,6 +135,7 @@ require ( github.com/pires/go-proxyproto v0.6.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/sftp v1.13.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -173,7 +177,6 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/mock v0.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.14.0 // indirect golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect diff --git a/go.sum b/go.sum index ac1477585..fd615c608 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,6 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= -cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= @@ -330,8 +328,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -355,23 +351,15 @@ github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8q github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -459,6 +447,7 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -499,6 +488,8 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= +github.com/melbahja/goph v1.4.0 h1:z0PgDbBFe66lRYl3v5dGb9aFgPy0kotuQ37QOwSQFqs= +github.com/melbahja/goph v1.4.0/go.mod h1:uG+VfK2Dlhk+O32zFrRlc3kYKTlV6+BtvPWd/kK7U68= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= @@ -564,6 +555,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posthog/posthog-go v0.0.0-20221221115252-24dfed35d71a h1:Ey0XWvrg6u6hyIn1Kd/jCCmL+bMv9El81tvuGBbxZGg= @@ -745,10 +738,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= @@ -855,8 +849,6 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -872,8 +864,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1064,8 +1054,6 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= -google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs= google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1116,16 +1104,10 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -1150,7 +1132,6 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 390aec87b..f213e072b 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -44,6 +44,13 @@ const ( MaxNumOfLogFiles = 5 RetainOldFiles = 0 // retain all old log files + SSHScriptTimeout = 120 * time.Second + SSHFileOpsTimeout = 30 * time.Second + SSHPOSTTimeout = 10 * time.Second + SSHSleepBetweenChecks = 1 * time.Second + SSHScriptLogFilter = "_AvalancheCLI_LOG_" + SSHShell = "/bin/bash" + ANRRequestTimeout = 3 * time.Minute APIRequestTimeout = 30 * time.Second @@ -153,25 +160,28 @@ const ( OldMetricsConfigFileName = ".avalanche-cli/config" DefaultConfigFileName = ".avalanche-cli/config.json" - AWSCloudService = "Amazon Web Services" - GCPCloudService = "Google Cloud Platform" - AnsibleSSHUser = "ubuntu" - AWSNodeAnsiblePrefix = "aws_node" - GCPNodeAnsiblePrefix = "gcp_node" - CustomVMDir = "vms" - GCPStaticIPPrefix = "static-ip" - AvaLabsOrg = "ava-labs" - AvalancheGoRepoName = "avalanchego" - SubnetEVMRepoName = "subnet-evm" - CliRepoName = "avalanche-cli" - UpgradeAvalancheGoPlaybook = "playbook/upgradeAvalancheGo.yml" - UpgradeSubnetEVMPlaybook = "playbook/upgradeSubnetEVM.yml" - StopNodePlaybook = "playbook/stopNode.yml" - StartNodePlaybook = "playbook/startNode.yml" - GetNewSubnetEVMPlaybook = "playbook/getNewSubnetEVMRelease.yml" - SubnetEVMReleaseURL = "https://github.com/ava-labs/subnet-evm/releases/download/%s/%s" - SubnetEVMArchive = "subnet-evm_%s_linux_amd64.tar.gz" - SubnetEVMBinaryPath = "/home/ubuntu/.avalanchego/plugins/%s" + AWSCloudService = "Amazon Web Services" + GCPCloudService = "Google Cloud Platform" + AnsibleSSHUser = "ubuntu" + AWSNodeAnsiblePrefix = "aws_node" + GCPNodeAnsiblePrefix = "gcp_node" + CustomVMDir = "vms" + GCPStaticIPPrefix = "static-ip" + AvaLabsOrg = "ava-labs" + AvalancheGoRepoName = "avalanchego" + SubnetEVMRepoName = "subnet-evm" + CliRepoName = "avalanche-cli" + UpgradeAvalancheGoPlaybook = "playbook/upgradeAvalancheGo.yml" + UpgradeSubnetEVMPlaybook = "playbook/upgradeSubnetEVM.yml" + StopNodePlaybook = "playbook/stopNode.yml" + StartNodePlaybook = "playbook/startNode.yml" + GetNewSubnetEVMPlaybook = "playbook/getNewSubnetEVMRelease.yml" + SubnetEVMReleaseURL = "https://github.com/ava-labs/subnet-evm/releases/download/%s/%s" + SubnetEVMArchive = "subnet-evm_%s_linux_amd64.tar.gz" + CloudNodeConfigBasePath = "/home/ubuntu/.avalanchego/" + CloudNodeSubnetEvmBinaryPath = "/home/ubuntu/.avalanchego/plugins/%s" + CloudNodeStakingPath = "/home/ubuntu/.avalanchego/staking/" + CloudNodeConfigPath = "/home/ubuntu/.avalanchego/configs/" AvalancheGoInstallDir = "avalanchego" SubnetEVMInstallDir = "subnet-evm" diff --git a/pkg/models/host.go b/pkg/models/host.go index ac580c8a8..29194bbad 100644 --- a/pkg/models/host.go +++ b/pkg/models/host.go @@ -3,10 +3,22 @@ package models import ( + "bytes" + "context" "fmt" + "net" + "os" + "path/filepath" "strings" + "time" "github.com/ava-labs/avalanche-cli/pkg/constants" + "github.com/melbahja/goph" + "golang.org/x/crypto/ssh" +) + +const ( + maxResponseSize = 102400 // 100KB should be enough to read the avalanchego response ) type Host struct { @@ -15,9 +27,160 @@ type Host struct { SSHUser string SSHPrivateKeyPath string SSHCommonArgs string + Connection *HostConnection +} + +type HostConnection struct { + Client *goph.Client + Ctx context.Context + ctxCancel context.CancelFunc +} + +func NewHostConnection(h Host, timeout time.Duration) *HostConnection { + p := new(HostConnection) + if timeout == 0 { + timeout = constants.SSHScriptTimeout + } + p.Ctx, p.ctxCancel = context.WithTimeout(context.Background(), timeout) + auth, err := goph.Key(h.SSHPrivateKeyPath, "") + if err != nil { + return nil + } + cl, err := goph.NewConn(&goph.Config{ + User: h.SSHUser, + Addr: h.IP, + Port: 22, + Auth: auth, + Timeout: timeout, + // #nosec G106 + Callback: ssh.InsecureIgnoreHostKey(), // we don't verify host key ( similar to ansible) + }) + if err != nil { + return nil + } else { + p.Client = cl + } + return p +} + +// GetCloudID returns the node ID of the host. +func (h *Host) GetCloudID() string { + _, cloudID, _ := HostAnsibleIDToCloudID(h.NodeID) + return cloudID +} + +// Connect starts a new SSH connection with the provided private key. +func (h *Host) Connect(timeout time.Duration) error { + h.Connection = NewHostConnection(*h, timeout) + if !h.Connected() { + return fmt.Errorf("failed to connect to host %s", h.IP) + } + return nil +} + +func (h *Host) Connected() bool { + return h.Connection != nil +} + +func (h *Host) Disconnect() error { + if !h.Connected() { + return nil + } + return h.Connection.Client.Close() +} + +// Upload uploads a local file to a remote file on the host. +func (h *Host) Upload(localFile string, remoteFile string) error { + if !h.Connected() { + if err := h.Connect(constants.SSHFileOpsTimeout); err != nil { + return err + } + } + return h.Connection.Client.Upload(localFile, remoteFile) +} + +// Download downloads a file from the remote server to the local machine. +func (h *Host) Download(remoteFile string, localFile string) error { + if !h.Connected() { + if err := h.Connect(constants.SSHScriptTimeout); err != nil { + return err + } + } + if err := os.MkdirAll(filepath.Dir(localFile), os.ModePerm); err != nil { + return err + } + return h.Connection.Client.Download(remoteFile, localFile) } -func (h Host) GetAnsibleInventoryRecord() string { +// MkdirAll creates a folder on the remote server. +func (h *Host) MkdirAll(remoteDir string) error { + if !h.Connected() { + if err := h.Connect(constants.SSHScriptTimeout); err != nil { + return err + } + } + sftp, err := h.Connection.Client.NewSftp() + if err != nil { + return err + } + defer sftp.Close() + return sftp.MkdirAll(remoteDir) +} + +// Command executes a shell command on a remote host. +func (h *Host) Command(script string, env []string, ctx context.Context) ([]byte, error) { + if !h.Connected() { + if err := h.Connect(constants.SSHScriptTimeout); err != nil { + return nil, err + } + } + if h.Connected() { + cmd, err := h.Connection.Client.CommandContext(ctx, constants.SSHShell, script) + if err != nil { + return nil, err + } + if env != nil { + cmd.Env = env + } + return cmd.CombinedOutput() + } else { + return nil, fmt.Errorf("failed to connect to host %s", h.IP) + } +} + +// Forward forwards the TCP connection to a remote address. +func (h *Host) Forward(httpRequest string) ([]byte, []byte, error) { + if !h.Connected() { + if err := h.Connect(constants.SSHPOSTTimeout); err != nil { + return nil, nil, err + } + } + avalancheGoEndpoint := strings.TrimPrefix(constants.LocalAPIEndpoint, "http://") + avalancheGoAddr, err := net.ResolveTCPAddr("tcp", avalancheGoEndpoint) + if err != nil { + return nil, nil, err + } + proxy, err := h.Connection.Client.DialTCP("tcp", nil, avalancheGoAddr) + if err != nil { + return nil, nil, fmt.Errorf("unable to port forward to %s via %s", h.Connection.Client.Conn.RemoteAddr(), "ssh") + } + defer proxy.Close() + // send request to server + _, err = proxy.Write([]byte(httpRequest)) + if err != nil { + return nil, nil, err + } + // Read and print the server's response + response := make([]byte, maxResponseSize) + responseLength, err := proxy.Read(response) + if err != nil { + return nil, nil, err + } + header, body := SplitHTTPResponse(response[:responseLength]) + return header, body, nil +} + +func (h *Host) GetAnsibleInventoryRecord() string { return strings.Join([]string{ h.NodeID, fmt.Sprintf("ansible_host=%s", h.IP), @@ -45,3 +208,57 @@ func HostAnsibleIDToCloudID(hostAnsibleID string) (string, string, error) { } return "", "", fmt.Errorf("unknown cloud service prefix in %s", hostAnsibleID) } + +// WaitForSSHPort waits for the SSH port to become available on the host. +func (h *Host) WaitForSSHPort(timeout time.Duration) error { + start := time.Now() + deadline := start.Add(timeout) + for { + if time.Now().After(deadline) { + return fmt.Errorf("timeout: SSH port %d on host %s is not available after %vs", constants.SSHTCPPort, h.IP, timeout.Seconds()) + } + if _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", h.IP, constants.SSHTCPPort), time.Second); err == nil { + return nil + } + time.Sleep(constants.SSHSleepBetweenChecks) + } +} + +// WaitForSSHShell waits for the SSH shell to be available on the host within the specified timeout. +func (h *Host) WaitForSSHShell(timeout time.Duration) error { + if err := h.WaitForSSHPort(timeout); err != nil { + return err + } + start := time.Now() + deadline := start.Add(timeout) + for { + if time.Now().After(deadline) { + return fmt.Errorf("timeout: SSH shell on host %s is not available after %ds", h.IP, int(timeout.Seconds())) + } + if err := h.Connect(timeout); err != nil { + time.Sleep(constants.SSHSleepBetweenChecks) + continue + } + if h.Connected() { + output, err := h.Command("echo", nil, context.Background()) + if err == nil || len(output) > 0 { + return nil + } + } + time.Sleep(constants.SSHSleepBetweenChecks) + } +} + +// splitHTTPResponse splits an HTTP response into headers and body. +func SplitHTTPResponse(response []byte) ([]byte, []byte) { + // Find the position of the double line break separating the headers and the body + doubleLineBreak := []byte{'\r', '\n', '\r', '\n'} + index := bytes.Index(response, doubleLineBreak) + if index == -1 { + return nil, response + } + // Split the response into headers and body + headers := response[:index] + body := response[index+len(doubleLineBreak):] + return headers, body +} diff --git a/pkg/models/result.go b/pkg/models/result.go new file mode 100644 index 000000000..8bf90e2ff --- /dev/null +++ b/pkg/models/result.go @@ -0,0 +1,84 @@ +// Copyright (C) 2022, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. +package models + +import "sync" + +type NodeResult struct { + NodeID string + Value interface{} + Err error +} +type NodeResults struct { + Results []NodeResult + Lock sync.Mutex +} + +func (nr *NodeResults) AddResult(nodeID string, value interface{}, err error) { + nr.Lock.Lock() + defer nr.Lock.Unlock() + nr.Results = append(nr.Results, NodeResult{ + NodeID: nodeID, + Value: value, + Err: err, + }) +} + +func (nr *NodeResults) GetResults() []NodeResult { + nr.Lock.Lock() + defer nr.Lock.Unlock() + return nr.Results +} + +func (nr *NodeResults) Len() int { + nr.Lock.Lock() + defer nr.Lock.Unlock() + return len(nr.Results) +} + +func (nr *NodeResults) GetNodeList() []string { + nr.Lock.Lock() + defer nr.Lock.Unlock() + nodes := []string{} + for _, node := range nr.Results { + nodes = append(nodes, node.NodeID) + } + return nodes +} + +func (nr *NodeResults) GetErroHostMap() map[string]error { + nr.Lock.Lock() + defer nr.Lock.Unlock() + hostErrors := make(map[string]error) + for _, node := range nr.Results { + if node.Err != nil { + hostErrors[node.NodeID] = node.Err + } + } + return hostErrors +} + +func (nr *NodeResults) HasNodeIDWithError(nodeID string) bool { + nr.Lock.Lock() + defer nr.Lock.Unlock() + for _, node := range nr.Results { + if node.NodeID == nodeID && node.Err != nil { + return true + } + } + return false +} + +func (nr *NodeResults) HasErrors() bool { + return len(nr.GetErroHostMap()) > 0 +} + +func (nr *NodeResults) GetErroHosts() []string { + var nodes []string + for _, node := range nr.Results { + if node.Err != nil { + nodes = append(nodes, node.NodeID) + } + } + return nodes +} diff --git a/pkg/ssh/shell/setupBuildEnv.sh b/pkg/ssh/shell/setupBuildEnv.sh new file mode 100755 index 000000000..d9251d508 --- /dev/null +++ b/pkg/ssh/shell/setupBuildEnv.sh @@ -0,0 +1,22 @@ +#!/ust/bin/env bash +#name:TASK [install gcc if not available] +gcc --version || DEBIAN_FRONTEND=noninteractive sudo apt-get -y install gcc +#name:TASK [install go] +install_go() { + GOFILE=go{{ .GoVersion }}.linux-amd64.tar.gz + cd ~ + sudo rm -rf $GOFILE go + wget -nv https://go.dev/dl/$GOFILE + tar xfz $GOFILE + echo >> ~/.bashrc + echo export PATH=\$PATH:~/go/bin:~/bin >> ~/.bashrc + echo export CGO_ENABLED=1 >> ~/.bashrc +} +go version || install_go +#name:TASK [install rust] +install_rust() { + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y + echo >> ~/.bashrc + echo export PATH=\$PATH:~/.cargo/bin >> ~/.bashrc +} +cargo version || install_rust diff --git a/pkg/ssh/shell/setupCLIFromSource.sh b/pkg/ssh/shell/setupCLIFromSource.sh new file mode 100644 index 000000000..b6d5f0f2b --- /dev/null +++ b/pkg/ssh/shell/setupCLIFromSource.sh @@ -0,0 +1,8 @@ +#!/ust/bin/env bash +#name:TASK [install avalanche-cli from source] +cd ~ +rm -rf avalanche-cli +git clone --single-branch -b {{ .CliBranch }} https://github.com/ava-labs/avalanche-cli +cd avalanche-cli +bash -i -c ./scripts/build.sh +cp bin/avalanche ~/bin/avalanche diff --git a/pkg/ssh/shell/setupDevnet.sh b/pkg/ssh/shell/setupDevnet.sh new file mode 100644 index 000000000..7da789d12 --- /dev/null +++ b/pkg/ssh/shell/setupDevnet.sh @@ -0,0 +1,8 @@ +#!/ust/bin/env bash +#name:TASK [stop node] +sudo systemctl stop avalanchego +#name:TASK [remove previous avalanchego db and logs] +rm -rf /home/ubuntu/.avalanchego/db/ +rm -rf /home/ubuntu/.avalanchego/logs/ +#name:TASK [start node] +sudo systemctl start avalanchego diff --git a/pkg/ssh/shell/setupNode.sh b/pkg/ssh/shell/setupNode.sh new file mode 100644 index 000000000..8d292155b --- /dev/null +++ b/pkg/ssh/shell/setupNode.sh @@ -0,0 +1,22 @@ +#!/ust/bin/env bash +#name:TASK [update apt data and install dependencies] +DEBIAN_FRONTEND=noninteractive sudo apt-get -y update +DEBIAN_FRONTEND=noninteractive sudo apt-get -y install wget curl git +#name:TASK [create .avalanche-cli .avalanchego dirs] +mkdir -p .avalanche-cli .avalanchego/staking +#name:TASK [get avalanche go script] +wget -nd -m https://raw.githubusercontent.com/ava-labs/avalanche-docs/master/scripts/avalanchego-installer.sh +#name:TASK [modify permissions] +chmod 755 avalanchego-installer.sh +#name:TASK [call avalanche go install script] +./avalanchego-installer.sh --ip static --rpc private --state-sync on --fuji --version {{ .AvalancheGoVersion }} +#name:TASK [get avalanche cli install script] +wget -nd -m https://raw.githubusercontent.com/ava-labs/avalanche-cli/main/scripts/install.sh +#name:TASK [modify permissions] +chmod 755 install.sh +#name:TASK [run install script] +./install.sh -n +{{if .IsDevNet}} +#name:TASK [stop avalanchego in case of devnet] +sudo systemctl stop avalanchego +{{end}} diff --git a/pkg/ssh/shell/trackSubnet.sh b/pkg/ssh/shell/trackSubnet.sh new file mode 100644 index 000000000..4915e24f0 --- /dev/null +++ b/pkg/ssh/shell/trackSubnet.sh @@ -0,0 +1,7 @@ +#!/ust/bin/env bash +#name:{{ .Log }}TASK [import subnet] +/home/ubuntu/bin/avalanche subnet import file {{ .SubnetExportFileName }} --force +#name:{{ .Log }}TASK [avalanche join subnet] +/home/ubuntu/bin/avalanche subnet join {{ .SubnetName }} --fuji --avalanchego-config /home/ubuntu/.avalanchego/configs/node.json --plugin-dir /home/ubuntu/.avalanchego/plugins --force-write +#name:{{ .Log }}TASK [restart node - restart avalanchego] +sudo systemctl restart avalanchego diff --git a/pkg/ssh/shell/updateSubnet.sh b/pkg/ssh/shell/updateSubnet.sh new file mode 100644 index 000000000..b49994a46 --- /dev/null +++ b/pkg/ssh/shell/updateSubnet.sh @@ -0,0 +1,9 @@ +#!/ust/bin/env bash +#name:TASK [stop node - stop avalanchego] +sudo systemctl stop avalanchego +#name:TASK [import subnet] +/home/ubuntu/bin/avalanche subnet import file {{ .SubnetExportFileName }} --force +#name:TASK [avalanche join subnet] +/home/ubuntu/bin/avalanche subnet join {{ .SubnetName }} --fuji --avalanchego-config /home/ubuntu/.avalanchego/configs/node.json --plugin-dir /home/ubuntu/.avalanchego/plugins --force-write +#name:TASK [restart node - start avalanchego] +sudo systemctl start avalanchego diff --git a/pkg/ssh/ssh.go b/pkg/ssh/ssh.go new file mode 100644 index 000000000..20747ee7a --- /dev/null +++ b/pkg/ssh/ssh.go @@ -0,0 +1,172 @@ +// Copyright (C) 2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. +package ssh + +import ( + "bytes" + "embed" + "fmt" + "net/url" + "path/filepath" + "text/template" + + "github.com/ava-labs/avalanche-cli/pkg/constants" + "github.com/ava-labs/avalanche-cli/pkg/models" + "github.com/ava-labs/avalanche-cli/pkg/ux" +) + +type scriptInputs struct { + AvalancheGoVersion string + SubnetExportFileName string + SubnetName string + GoVersion string + CliBranch string + IsDevNet bool +} + +//go:embed shell/*.sh +var script embed.FS + +// scriptLog formats the given line of a script log with the provided nodeID. +func scriptLog(nodeID string, line string) string { + return fmt.Sprintf("[%s] %s", nodeID, line) +} + +// RunOverSSH runs provided script path over ssh. +// This script can be template as it will be rendered using scriptInputs vars +func RunOverSSH(scriptDesc string, host models.Host, scriptPath string, templateVars scriptInputs) error { + shellScript, err := script.ReadFile(scriptPath) + if err != nil { + return err + } + + var script bytes.Buffer + t, err := template.New(scriptDesc).Parse(string(shellScript)) + if err != nil { + return err + } + err = t.Execute(&script, templateVars) + if err != nil { + return err + } + ux.Logger.PrintToUser(scriptLog(host.NodeID, scriptDesc)) + if _, err := host.Command(script.String(), nil, host.Connection.Ctx); err != nil { + return err + } + return nil +} + +func PostOverSSH(host models.Host, path string, requestBody string) ([]byte, error) { + if path == "" { + path = "/ext/info" + } + localhost, err := url.Parse(constants.LocalAPIEndpoint) + if err != nil { + return nil, err + } + requestHeaders := fmt.Sprintf("POST %s HTTP/1.1\r\n"+ + "Host: %s\r\n"+ + "Content-Length: %d\r\n"+ + "Content-Type: application/json\r\n\r\n", path, localhost.Host, len(requestBody)) + httpRequest := requestHeaders + requestBody + // ignore response header + _, responseBody, err := host.Forward(httpRequest) + if err != nil { + return nil, err + } + return responseBody, nil +} + +// RunSSHSetupNode runs script to setup node +func RunSSHSetupNode(host models.Host, configPath, avalancheGoVersion string, isDevNet bool) error { + // name: setup node + if err := RunOverSSH("Setup Node", host, "shell/setupNode.sh", scriptInputs{AvalancheGoVersion: avalancheGoVersion, IsDevNet: isDevNet}); err != nil { + return err + } + // name: copy metrics config to cloud server + return host.Upload(configPath, filepath.Join(constants.CloudNodeConfigBasePath, filepath.Base(configPath))) +} + +// RunSSHSetupDevNet runs script to setup devnet +func RunSSHSetupDevNet(host models.Host, nodeInstanceDirPath string) error { + if err := host.MkdirAll(constants.CloudNodeConfigPath); err != nil { + return err + } + if err := host.Upload(filepath.Join(nodeInstanceDirPath, constants.GenesisFileName), filepath.Join(constants.CloudNodeConfigPath, constants.GenesisFileName)); err != nil { + return err + } + if err := host.Upload(filepath.Join(nodeInstanceDirPath, constants.NodeFileName), filepath.Join(constants.CloudNodeConfigPath, constants.NodeFileName)); err != nil { + return err + } + // name: setup devnet + return RunOverSSH("Setup DevNet", host, "shell/setupDevnet.sh", scriptInputs{}) +} + +// RunSSHUploadStakingFiles uploads staking files to a remote host via SSH. +func RunSSHUploadStakingFiles(host models.Host, nodeInstanceDirPath string) error { + if err := host.MkdirAll(constants.CloudNodeStakingPath); err != nil { + return err + } + if err := host.Upload(filepath.Join(nodeInstanceDirPath, constants.StakerCertFileName), filepath.Join(constants.CloudNodeStakingPath, constants.StakerCertFileName)); err != nil { + return err + } + if err := host.Upload(filepath.Join(nodeInstanceDirPath, constants.StakerKeyFileName), filepath.Join(constants.CloudNodeStakingPath, constants.StakerKeyFileName)); err != nil { + return err + } + return host.Upload(filepath.Join(nodeInstanceDirPath, constants.BLSKeyFileName), filepath.Join(constants.CloudNodeStakingPath, constants.BLSKeyFileName)) +} + +// RunSSHExportSubnet exports deployed Subnet from local machine to cloud server +func RunSSHExportSubnet(host models.Host, exportPath, cloudServerSubnetPath string) error { + // name: copy exported subnet VM spec to cloud server + return host.Upload(exportPath, cloudServerSubnetPath) +} + +// RunSSHExportSubnet exports deployed Subnet from local machine to cloud server +// targets a specific host ansibleHostID in ansible inventory file +func RunSSHTrackSubnet(host models.Host, subnetName, importPath string) error { + return RunOverSSH("Track Subnet", host, "shell/trackSubnet.sh", scriptInputs{SubnetName: subnetName, SubnetExportFileName: importPath}) +} + +// RunSSHUpdateSubnet runs avalanche subnet join in cloud server using update subnet info +func RunSSHUpdateSubnet(host models.Host, subnetName, importPath string) error { + return RunOverSSH("Track Subnet", host, "shell/updateSubnet.sh", scriptInputs{SubnetName: subnetName, SubnetExportFileName: importPath}) +} + +// RunSSHSetupBuildEnv installs gcc, golang, rust and etc +func RunSSHSetupBuildEnv(host models.Host) error { + return RunOverSSH("Setup Build Env", host, "shell/setupBuildEnv.sh", scriptInputs{GoVersion: constants.BuildEnvGolangVersion}) +} + +// RunSSHSetupCLIFromSource installs any CLI branch from source +func RunSSHSetupCLIFromSource(host models.Host, cliBranch string) error { + return RunOverSSH("Setup CLI From Source", host, "shell/setupCLIFromSource.sh", scriptInputs{CliBranch: cliBranch}) +} + +// RunSSHCheckAvalancheGoVersion checks node avalanchego version +func RunSSHCheckAvalancheGoVersion(host models.Host) ([]byte, error) { + // Craft and send the HTTP POST request + requestBody := "{\"jsonrpc\":\"2.0\", \"id\":1,\"method\" :\"info.getNodeVersion\"}" + return PostOverSSH(host, "", requestBody) +} + +// RunSSHCheckBootstrapped checks if node is bootstrapped to primary network +func RunSSHCheckBootstrapped(host models.Host) ([]byte, error) { + // Craft and send the HTTP POST request + requestBody := "{\"jsonrpc\":\"2.0\", \"id\":1,\"method\" :\"info.isBootstrapped\", \"params\": {\"chain\":\"X\"}}" + return PostOverSSH(host, "", requestBody) +} + +// RunSSHGetNodeID reads nodeID from avalanchego +func RunSSHGetNodeID(host models.Host) ([]byte, error) { + // Craft and send the HTTP POST request + requestBody := "{\"jsonrpc\":\"2.0\", \"id\":1,\"method\" :\"info.getNodeID\"}" + return PostOverSSH(host, "", requestBody) +} + +// SubnetSyncStatus checks if node is synced to subnet +func RunSSHSubnetSyncStatus(host models.Host, blockchainID string) ([]byte, error) { + // Craft and send the HTTP POST request + requestBody := fmt.Sprintf("{\"jsonrpc\":\"2.0\", \"id\":1,\"method\" :\"platform.getBlockchainStatus\", \"params\": {\"blockchainID\":\"%s\"}}", blockchainID) + return PostOverSSH(host, "/ext/bc/P", requestBody) +}