Skip to content

Commit

Permalink
first PR to depricate ansible (#1186)
Browse files Browse the repository at this point in the history 
* first PR to depricate ansible

* rm unused func

* get error and pass it to user UI

* address part of the felipe feedback

* update ssh shell

* more feedback

* use slices

* fix go mod

* make ssh to reuse tcp connections

* fix lint

* rm debug

* devnet via ssh

* rm unused code

* nit

* cleanup

* fix lint

* adddress missed feedback

* nit

* fix setup devnet

* added comment

* stop in case of error

* stop avago in case of devnet

* fix lint
  • Loading branch information
@arturrez
arturrez authored Nov 15, 2023
1 parent 2428aeb commit bab7aec
Show file tree
Hide file tree
Showing 15 changed files with 745 additions and 123 deletions.
178 changes: 114 additions & 64 deletions cmd/nodecmd/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,31 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"time"
"sync"

"github.com/ava-labs/avalanche-cli/cmd/subnetcmd"
"github.com/ava-labs/avalanche-cli/pkg/ansible"
awsAPI "github.com/ava-labs/avalanche-cli/pkg/aws"
gcpAPI "github.com/ava-labs/avalanche-cli/pkg/gcp"
"github.com/ava-labs/avalanche-cli/pkg/ssh"
"github.com/ava-labs/avalanche-cli/pkg/terraform"
"github.com/ava-labs/avalanche-cli/pkg/utils"
"github.com/ava-labs/avalanche-cli/pkg/vm"
"github.com/ava-labs/avalanchego/ids"
"github.com/ava-labs/avalanchego/staking"
"golang.org/x/exp/slices"

"github.com/ava-labs/avalanche-cli/pkg/constants"
"github.com/ava-labs/avalanche-cli/pkg/models"

"github.com/ava-labs/avalanche-cli/pkg/ux"
"github.com/spf13/cobra"

"golang.org/x/sync/errgroup"
)

const (
Expand Down Expand Up @@ -217,40 +217,90 @@ func createNodes(_ *cobra.Command, args []string) error {
if err != nil {
return err
}

time.Sleep(30 * time.Second)

inventoryPath := app.GetAnsibleInventoryDirPath(clusterName)
avalancheGoVersion, err := getAvalancheGoVersion()
if err != nil {
return err
}
inventoryPath := app.GetAnsibleInventoryDirPath(clusterName)
if err = ansible.CreateAnsibleHostInventory(inventoryPath, cloudConfig.CertFilePath, cloudService, publicIPMap); err != nil {
return err
}
if err := updateAnsiblePublicIPs(clusterName); err != nil {
return err
}
allHosts, err := ansible.GetInventoryFromAnsibleInventoryFile(inventoryPath)
if err != nil {
return err
}
hosts := utils.Filter(allHosts, func(h models.Host) bool { return slices.Contains(cloudConfig.InstanceIDs, h.GetCloudID()) })
// waiting for all nodes to become accessible
failedHosts := waitForHosts(hosts)
if failedHosts.Len() > 0 {
for _, result := range failedHosts.GetResults() {
ux.Logger.PrintToUser("Instance %s failed to provision with error %s. Please check instance logs for more information", result.NodeID, result.Err)
}
return fmt.Errorf("failed to provision node(s) %s", failedHosts.GetNodeList())
}

ux.Logger.PrintToUser("Installing AvalancheGo and Avalanche-CLI and starting bootstrap process on the newly created Avalanche node(s) ...")
ansibleHostIDs, err := utils.MapWithError(cloudConfig.InstanceIDs, func(s string) (string, error) { return models.HostCloudIDToAnsibleID(cloudService, s) })
if err != nil {
return err
}
createdAnsibleHostIDs := strings.Join(ansibleHostIDs, ",")
if err = runAnsible(inventoryPath, network, avalancheGoVersion, clusterName, createdAnsibleHostIDs); err != nil {
return err
ux.Logger.PrintToUser("Installing AvalancheGo and Avalanche-CLI and starting bootstrap process on the newly created Avalanche node(s) ...")
wg := sync.WaitGroup{}
wgResults := models.NodeResults{}
for _, host := range hosts {
wg.Add(1)
go func(nodeResults *models.NodeResults, host models.Host) {
defer wg.Done()
if err := host.Connect(constants.SSHScriptTimeout); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
defer func() {
if err := host.Disconnect(); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
}
}()
if err := provideStakingCertAndKey(host); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
if err := ssh.RunSSHSetupNode(host, app.Conf.GetConfigPath(), avalancheGoVersion, network.Kind == models.Devnet); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
if err := ssh.RunSSHSetupBuildEnv(host); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
}(&wgResults, host)
}
if err = setupBuildEnv(inventoryPath, createdAnsibleHostIDs); err != nil {
return err
wg.Wait()
ux.Logger.PrintToUser("======================================")
ux.Logger.PrintToUser("AVALANCHE NODE(S) STATUS")
ux.Logger.PrintToUser("======================================")
ux.Logger.PrintToUser("")
for _, node := range hosts {
if wgResults.HasNodeIDWithError(node.NodeID) {
ux.Logger.PrintToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErroHostMap()[node.NodeID])
} else {
ux.Logger.PrintToUser("Node %s is CREATED", node.NodeID)
}
}

if network.Kind == models.Devnet {
ux.Logger.PrintToUser("Setting up Devnet ...")
if err := setupDevnet(clusterName); err != nil {
return err
}
}

printResults(cloudConfig, publicIPMap, ansibleHostIDs)
ux.Logger.PrintToUser("AvalancheGo and Avalanche-CLI installed and node(s) are bootstrapping!")
if wgResults.HasErrors() {
return fmt.Errorf("failed to deploy node(s) %s", wgResults.GetErroHosts())
} else {
printResults(cloudConfig, publicIPMap, ansibleHostIDs)
ux.Logger.PrintToUser("AvalancheGo and Avalanche-CLI installed and node(s) are bootstrapping!")
}
return nil
}

Expand Down Expand Up @@ -341,23 +391,6 @@ func setupAnsible(clusterName string) error {
return updateAnsiblePublicIPs(clusterName)
}

func runAnsible(inventoryPath string, network models.Network, avalancheGoVersion, clusterName, ansibleHostIDs string) error {
if err := setupAnsible(clusterName); err != nil {
return err
}
if err := distributeStakingCertAndKey(strings.Split(ansibleHostIDs, ","), inventoryPath); err != nil {
return err
}
return ansible.RunAnsiblePlaybookSetupNode(
app.Conf.GetConfigPath(),
app.GetAnsibleDir(),
inventoryPath,
avalancheGoVersion,
fmt.Sprint(network.Kind == models.Devnet),
ansibleHostIDs,
)
}

func setupBuildEnv(inventoryPath, ansibleHostIDs string) error {
ux.Logger.PrintToUser("Installing Custom VM build environment on the cloud server(s) ...")
ansibleTargetHosts := "all"
Expand Down Expand Up @@ -417,53 +450,52 @@ func generateNodeCertAndKeys(stakerCertFilePath, stakerKeyFilePath, blsKeyFilePa
return nodeID, nil
}

func distributeStakingCertAndKey(ansibleHostIDs []string, inventoryPath string) error {
ux.Logger.PrintToUser("Generating staking keys in local machine...")
eg := errgroup.Group{}
for _, ansibleInstanceID := range ansibleHostIDs {
_, instanceID, err := models.HostAnsibleIDToCloudID(ansibleInstanceID)
if err != nil {
return err
}
keyPath := filepath.Join(app.GetNodesDir(), instanceID)
eg.Go(func() error {
nodeID, err := generateNodeCertAndKeys(
filepath.Join(keyPath, constants.StakerCertFileName),
filepath.Join(keyPath, constants.StakerKeyFileName),
filepath.Join(keyPath, constants.BLSKeyFileName),
)
if err != nil {
ux.Logger.PrintToUser("Failed to generate staking keys for host %s", instanceID)
return err
} else {
ux.Logger.PrintToUser("Generated staking keys for host %s[%s] ", instanceID, nodeID.String())
}
return nil
})
}
if err := eg.Wait(); err != nil {
func provideStakingCertAndKey(host models.Host) error {
instanceID := host.GetCloudID()
keyPath := filepath.Join(app.GetNodesDir(), instanceID)
nodeID, err := generateNodeCertAndKeys(
filepath.Join(keyPath, constants.StakerCertFileName),
filepath.Join(keyPath, constants.StakerKeyFileName),
filepath.Join(keyPath, constants.BLSKeyFileName),
)
if err != nil {
ux.Logger.PrintToUser("Failed to generate staking keys for host %s", instanceID)
return err
} else {
ux.Logger.PrintToUser("Generated staking keys for host %s[%s] ", instanceID, nodeID.String())
}
ux.Logger.PrintToUser("Copying staking keys to remote machine(s)...")
return ansible.RunAnsiblePlaybookCopyStakingFiles(app.GetAnsibleDir(), strings.Join(ansibleHostIDs, ","), app.GetNodesDir(), inventoryPath)
return ssh.RunSSHUploadStakingFiles(host, keyPath)
}

func getIPAddress() (string, error) {
ipOutput, err := exec.Command("curl", "https://api.ipify.org?format=json").Output()
resp, err := http.Get("https://api.ipify.org?format=json")
if err != nil {
return "", err
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return "", errors.New("HTTP request failed")
}

body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}

var result map[string]interface{}
if err = json.Unmarshal(ipOutput, &result); err != nil {
if err := json.Unmarshal(body, &result); err != nil {
return "", err
}

ipAddress, ok := result["ip"].(string)
if ok {
if net.ParseIP(ipAddress) == nil {
return "", errors.New("invalid IP address")
}
return ipAddress, nil
}

return "", errors.New("no IP address found")
}

Expand Down Expand Up @@ -592,3 +624,21 @@ func printResults(cloudConfig CloudConfig, publicIPMap map[string]string, ansibl
ux.Logger.PrintToUser(fmt.Sprintf("Don't delete or replace your ssh private key file at %s as you won't be able to access your cloud server without it", cloudConfig.CertFilePath))
ux.Logger.PrintToUser("")
}

// waitForHosts waits for all hosts to become available via SSH.
func waitForHosts(hosts []models.Host) *models.NodeResults {
hostErrors := models.NodeResults{}
createdWaitGroup := sync.WaitGroup{}
for _, host := range hosts {
createdWaitGroup.Add(1)
go func(nodeResults *models.NodeResults, host models.Host) {
defer createdWaitGroup.Done()
if err := host.WaitForSSHShell(2 * constants.SSHFileOpsTimeout); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
}(&hostErrors, host)
}
createdWaitGroup.Wait()
return &hostErrors
}
51 changes: 40 additions & 11 deletions cmd/nodecmd/create_devnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"

"github.com/ava-labs/avalanche-cli/pkg/ansible"
"github.com/ava-labs/avalanche-cli/pkg/constants"
"github.com/ava-labs/avalanche-cli/pkg/key"
"github.com/ava-labs/avalanche-cli/pkg/models"
"github.com/ava-labs/avalanche-cli/pkg/ssh"
"github.com/ava-labs/avalanche-cli/pkg/utils"
"github.com/ava-labs/avalanche-cli/pkg/ux"
"github.com/ava-labs/avalanchego/config"
Expand Down Expand Up @@ -111,10 +113,8 @@ func setupDevnet(clusterName string) error {
if err := checkCluster(clusterName); err != nil {
return err
}
if err := setupAnsible(clusterName); err != nil {
return err
}
ansibleHostIDs, err := ansible.GetAnsibleHostsFromInventory(app.GetAnsibleInventoryDirPath(clusterName))
inventoryPath := app.GetAnsibleInventoryDirPath(clusterName)
ansibleHostIDs, err := ansible.GetAnsibleHostsFromInventory(inventoryPath)
if err != nil {
return err
}
Expand Down Expand Up @@ -188,16 +188,45 @@ func setupDevnet(clusterName string) error {
return err
}
}

// update node/s genesis + conf and start
if err := ansible.RunAnsiblePlaybookSetupDevnet(
app.GetAnsibleDir(),
strings.Join(ansibleHostIDs, ","),
app.GetNodesDir(),
app.GetAnsibleInventoryDirPath(clusterName),
); err != nil {
hosts, err := ansible.GetInventoryFromAnsibleInventoryFile(inventoryPath)
if err != nil {
return err
}
wg := sync.WaitGroup{}
wgResults := models.NodeResults{}
for _, host := range hosts {
wg.Add(1)
go func(nodeResults *models.NodeResults, host models.Host) {
defer wg.Done()
if err := host.Connect(constants.SSHScriptTimeout); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
defer func() {
if err := host.Disconnect(); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
}
}()
keyPath := filepath.Join(app.GetNodesDir(), host.GetCloudID())
if err := ssh.RunSSHSetupDevNet(host, keyPath); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
}(&wgResults, host)
}
wg.Wait()
for _, node := range hosts {
if wgResults.HasNodeIDWithError(node.NodeID) {
ux.Logger.PrintToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErroHostMap()[node.NodeID])
} else {
ux.Logger.PrintToUser("Node %s is SETUP as devnet", node.NodeID)
}
}
// stop execution if at least one node failed
if wgResults.HasErrors() {
return fmt.Errorf("failed to deploy node(s) %s", wgResults.GetErroHosts())
}

// update cluster config with network information
clustersConfig, err := app.LoadClustersConfig()
Expand Down
2 changes: 1 addition & 1 deletion cmd/nodecmd/upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func upgrade(_ *cobra.Command, args []string) error {
return err
}
for _, vmID := range upgradeInfo.SubnetEVMIDsToUpgrade {
subnetEVMBinaryPath := fmt.Sprintf(constants.SubnetEVMBinaryPath, vmID)
subnetEVMBinaryPath := fmt.Sprintf(constants.CloudNodeSubnetEvmBinaryPath, vmID)
if err = upgradeSubnetEVM(clusterName, subnetEVMBinaryPath, node, upgradeInfo.SubnetEVMVersion); err != nil {
return err
}
Expand Down
5 changes: 4 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/hashicorp/hcl/v2 v2.17.0
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
github.com/manifoldco/promptui v0.9.0
github.com/melbahja/goph v1.4.0
github.com/olekukonko/tablewriter v0.0.5
github.com/onsi/ginkgo/v2 v2.12.0
github.com/onsi/gomega v1.27.10
Expand All @@ -26,6 +27,7 @@ require (
github.com/stretchr/testify v1.8.4
github.com/zclconf/go-cty v1.13.0
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
golang.org/x/mod v0.12.0
golang.org/x/net v0.17.0
Expand Down Expand Up @@ -115,6 +117,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.15.15 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
Expand All @@ -132,6 +135,7 @@ require (
github.com/pires/go-proxyproto v0.6.2 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/sftp v1.13.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
Expand Down Expand Up @@ -173,7 +177,6 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/mock v0.2.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
Expand Down
Loading

0 comments on commit bab7aec

Please sign in to comment.