From 2c76e3f375fef92bdab110173cab7e6233d0ff13 Mon Sep 17 00:00:00 2001 From: sukantoraymond Date: Thu, 6 Feb 2025 16:12:39 -0500 Subject: [PATCH] Refactor node setup out of node create (#2522) * refactor node setup * fix lint * address comments * fix lint * address comments * address comments * address comment * fix merge --------- Signed-off-by: sukantoraymond --- cmd/nodecmd/create.go | 148 +++++++------- cmd/nodecmd/create_devnet.go | 2 +- cmd/nodecmd/local.go | 70 +++---- cmd/nodecmd/node.go | 2 + cmd/nodecmd/setup.go | 233 +++++++++++++++++++++++ cmd/nodecmd/wiz.go | 2 +- pkg/application/app.go | 4 + pkg/constants/constants.go | 3 +- pkg/models/host.go | 1 + pkg/models/host_test.go | 4 +- pkg/models/result.go | 4 +- pkg/utils/ssh.go | 4 +- tests/e2e/testcases/node/create/suite.go | 5 +- tests/e2e/utils/helpers.go | 11 ++ 14 files changed, 373 insertions(+), 120 deletions(-) create mode 100644 cmd/nodecmd/setup.go diff --git a/cmd/nodecmd/create.go b/cmd/nodecmd/create.go index b00c6a34e..0fc18d0d0 100644 --- a/cmd/nodecmd/create.go +++ b/cmd/nodecmd/create.go @@ -31,6 +31,7 @@ import ( "github.com/ava-labs/avalanche-cli/pkg/utils" "github.com/ava-labs/avalanche-cli/pkg/ux" "github.com/ava-labs/avalanche-cli/pkg/vm" + sdkUtils "github.com/ava-labs/avalanche-cli/sdk/utils" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/staking" "github.com/ava-labs/avalanchego/utils/logging" @@ -661,7 +662,6 @@ func createNodes(cmd *cobra.Command, args []string) error { } return fmt.Errorf("failed to provision node(s) %s", failedHosts.GetNodeList()) } - ux.Logger.PrintToUser("Starting bootstrap process on the newly created Avalanche node(s)...") wg := sync.WaitGroup{} wgResults := models.NodeResults{} spinSession := ux.NewUserSpinner() @@ -672,6 +672,7 @@ func createNodes(cmd *cobra.Command, args []string) error { } startTime := time.Now() if addMonitoring { + spinSession := ux.NewUserSpinner() if len(monitoringHosts) != 1 { return fmt.Errorf("expected only one monitoring host, found %d", len(monitoringHosts)) } @@ -682,47 +683,47 @@ func createNodes(cmd *cobra.Command, args []string) error { go func(nodeResults *models.NodeResults, monitoringHost *models.Host) { defer wg.Done() if err := monitoringHost.Connect(0); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) return } - spinner := spinSession.SpinToUser(utils.ScriptLog(monitoringHost.NodeID, "Setup Monitoring")) + spinner := spinSession.SpinToUser(utils.ScriptLog(monitoringHost.IP, "Setup Monitoring")) if err = app.SetupMonitoringEnv(); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } if err = ssh.RunSSHSetupDockerService(monitoringHost); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } ux.Logger.Info("SetupMonitoringEnv RunSSHSetupDockerService completed") if err = ssh.RunSSHSetupMonitoringFolders(monitoringHost); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } ux.Logger.Info("RunSSHSetupMonitoringFolders completed") if err := ssh.RunSSHCopyMonitoringDashboards(monitoringHost, app.GetMonitoringDashboardDir()+"/"); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } ux.Logger.Info("RunSSHCopyMonitoringDashboards completed") if err := ssh.RunSSHSetupPrometheusConfig(monitoringHost, avalancheGoPorts, machinePorts, ltPorts); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } ux.Logger.Info("RunSSHSetupPrometheusConfig completed") if err := ssh.RunSSHSetupLokiConfig(monitoringHost, constants.AvalancheGoLokiPort); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } ux.Logger.Info("RunSSHSetupLokiConfig completed") if err := docker.ComposeSSHSetupMonitoring(monitoringHost); err != nil { - nodeResults.AddResult(monitoringHost.NodeID, nil, err) + nodeResults.AddResult(monitoringHost.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } @@ -730,68 +731,43 @@ func createNodes(cmd *cobra.Command, args []string) error { ux.SpinComplete(spinner) }(&wgResults, monitoringHost) } + wg.Wait() + spinSession.Stop() } for _, host := range hosts { - wg.Add(1) - go func(nodeResults *models.NodeResults, host *models.Host) { - defer wg.Done() - if err := host.Connect(0); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - return - } - if err := provideStakingCertAndKey(host); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - return - } - spinner := spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Setup Node")) - if err := ssh.RunSSHSetupNode(host, app.Conf.GetConfigPath()); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - ux.SpinFailWithError(spinner, "", err) - return - } - if err := ssh.RunSSHSetupDockerService(host); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - ux.SpinFailWithError(spinner, "", err) - return - } - ux.SpinComplete(spinner) - if addMonitoring { - cloudID := host.GetCloudID() - nodeID, err := getNodeID(app.GetNodeInstanceDirPath(cloudID)) - if err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - ux.SpinFailWithError(spinner, "", err) - return - } - if err = ssh.RunSSHSetupPromtailConfig(host, monitoringNodeConfig.PublicIPs[0], constants.AvalancheGoLokiPort, cloudID, nodeID.String(), ""); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - ux.SpinFailWithError(spinner, "", err) - return + publicAccessToHTTPPort := slices.Contains(cloudConfigMap.GetAllAPIInstanceIDs(), host.GetCloudID()) || publicHTTPPortAccess + host.APINode = publicAccessToHTTPPort + } + if err = setup(hosts, avalancheGoVersion, network); err != nil { + return err + } + if addMonitoring { + spinSession := ux.NewUserSpinner() + for _, host := range hosts { + wg.Add(1) + go func(nodeResults *models.NodeResults, host *models.Host) { + defer wg.Done() + spinner := spinSession.SpinToUser(utils.ScriptLog(host.IP, "Add Monitoring")) + if addMonitoring { + cloudID := host.GetCloudID() + nodeID, err := getNodeID(app.GetNodeInstanceDirPath(cloudID)) + if err != nil { + nodeResults.AddResult(host.IP, nil, err) + ux.SpinFailWithError(spinner, "", err) + return + } + if err = ssh.RunSSHSetupPromtailConfig(host, monitoringNodeConfig.PublicIPs[0], constants.AvalancheGoLokiPort, cloudID, nodeID.String(), ""); err != nil { + nodeResults.AddResult(host.IP, nil, err) + ux.SpinFailWithError(spinner, "", err) + return + } + ux.SpinComplete(spinner) } - ux.SpinComplete(spinner) - } - spinner = spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Setup AvalancheGo")) - // check if host is a API host - publicAccessToHTTPPort := slices.Contains(cloudConfigMap.GetAllAPIInstanceIDs(), host.GetCloudID()) || publicHTTPPortAccess - if err := docker.ComposeSSHSetupNode(host, - network, - avalancheGoVersion, - bootstrapIDs, - bootstrapIPs, - partialSync, - genesisPath, - upgradePath, - addMonitoring, - publicAccessToHTTPPort, - ); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) - ux.SpinFailWithError(spinner, "", err) - return - } - ux.SpinComplete(spinner) - }(&wgResults, host) + }(&wgResults, host) + } + wg.Wait() + spinSession.Stop() } - wg.Wait() ux.Logger.Info("Create and setup nodes time took: %s", time.Since(startTime)) spinSession.Stop() if network.Kind == models.Devnet { @@ -800,8 +776,8 @@ func createNodes(cmd *cobra.Command, args []string) error { } } for _, node := range hosts { - if wgResults.HasNodeIDWithError(node.NodeID) { - ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID]) + if wgResults.HasIDWithError(node.IP) { + ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.IP, wgResults.GetErrorHostMap()[node.IP]) } } @@ -1033,18 +1009,38 @@ func generateNodeCertAndKeys(stakerCertFilePath, stakerKeyFilePath, blsKeyFilePa } func provideStakingCertAndKey(host *models.Host) error { - instanceID := host.GetCloudID() - keyPath := filepath.Join(app.GetNodesDir(), instanceID) + keyPath := app.GetNodeStakingDir(host.IP) + if sdkUtils.DirExists(keyPath) && !overrideExisting { + yes, err := app.Prompt.CaptureNoYes(fmt.Sprintf("Directory %s alreday exists. Do you want to override it?", keyPath)) + if err != nil { + return err + } + if !yes { + return nil + } + } nodeID, err := generateNodeCertAndKeys( filepath.Join(keyPath, constants.StakerCertFileName), filepath.Join(keyPath, constants.StakerKeyFileName), filepath.Join(keyPath, constants.BLSKeyFileName), ) if err != nil { - ux.Logger.PrintToUser("Failed to generate staking keys for host %s", instanceID) + ux.Logger.PrintToUser("Failed to generate staking keys for host %s", host.IP) return err } else { - ux.Logger.GreenCheckmarkToUser("Generated staking keys for host %s[%s] ", instanceID, nodeID.String()) + ux.Logger.GreenCheckmarkToUser("Generated staking keys for host %s[%s] ", host.IP, nodeID.String()) + } + instanceID := host.GetCloudID() + if instanceID != "" { + if err := utils.FileCopy(filepath.Join(keyPath, constants.StakerCertFileName), filepath.Join(app.GetNodesDir(), instanceID, constants.StakerCertFileName)); err != nil { + return err + } + if err := utils.FileCopy(filepath.Join(keyPath, constants.StakerKeyFileName), filepath.Join(app.GetNodesDir(), instanceID, constants.StakerKeyFileName)); err != nil { + return err + } + if err := utils.FileCopy(filepath.Join(keyPath, constants.BLSKeyFileName), filepath.Join(app.GetNodesDir(), instanceID, constants.BLSKeyFileName)); err != nil { + return err + } } return ssh.RunSSHUploadStakingFiles(host, keyPath) } @@ -1214,9 +1210,9 @@ func waitForHosts(hosts []*models.Host) *models.NodeResults { createdWaitGroup.Add(1) go func(nodeResults *models.NodeResults, host *models.Host) { defer createdWaitGroup.Done() - spinner := spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Waiting for instance response")) + spinner := spinSession.SpinToUser(utils.ScriptLog(host.IP, "Waiting for instance response")) if err := host.WaitForSSHShell(constants.SSHServerStartTimeout); err != nil { - nodeResults.AddResult(host.NodeID, nil, err) + nodeResults.AddResult(host.IP, nil, err) ux.SpinFailWithError(spinner, "", err) return } diff --git a/cmd/nodecmd/create_devnet.go b/cmd/nodecmd/create_devnet.go index 08caae6bb..dfb9d1a68 100644 --- a/cmd/nodecmd/create_devnet.go +++ b/cmd/nodecmd/create_devnet.go @@ -273,7 +273,7 @@ func setupDevnet(clusterName string, hosts []*models.Host, apiNodeIPMap map[stri wg.Wait() ux.Logger.PrintLineSeparator() for _, node := range hosts { - if wgResults.HasNodeIDWithError(node.NodeID) { + if wgResults.HasIDWithError(node.NodeID) { ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID]) } else { nodeID, err := getNodeID(app.GetNodeInstanceDirPath(node.GetCloudID())) diff --git a/cmd/nodecmd/local.go b/cmd/nodecmd/local.go index 24b2f2075..9eecd828a 100644 --- a/cmd/nodecmd/local.go +++ b/cmd/nodecmd/local.go @@ -41,28 +41,30 @@ import ( var ( avalanchegoBinaryPath string - bootstrapIDs []string - bootstrapIPs []string - genesisPath string - upgradePath string - stakingTLSKeyPath string - stakingCertKeyPath string - stakingSignerKeyPath string - numNodes uint32 - nodeConfigPath string - partialSync bool - stakeAmount uint64 - rpcURL string - balanceAVAX float64 - remainingBalanceOwnerAddr string - disableOwnerAddr string - aggregatorLogLevel string - aggregatorLogToStdout bool - delegationFee uint16 - publicKey string - pop string - minimumStakeDuration uint64 - validatorManagerAddress string + bootstrapIDs []string + bootstrapIPs []string + genesisPath string + upgradePath string + stakingTLSKeyPath string + stakingCertKeyPath string + stakingSignerKeyPath string + numNodes uint32 + nodeConfigPath string + partialSync bool + stakeAmount uint64 + rpcURL string + balanceAVAX float64 + remainingBalanceOwnerAddr string + disableOwnerAddr string + aggregatorLogLevel string + aggregatorLogToStdout bool + delegationFee uint16 + publicKey string + pop string + minimumStakeDuration uint64 + latestAvagoReleaseVersion bool + latestAvagoPreReleaseVersion bool + validatorManagerAddress string ) // const snapshotName = "local_snapshot" @@ -104,8 +106,8 @@ You can check the bootstrapping status by running avalanche node status local. PersistentPostRun: handlePostRun, } networkoptions.AddNetworkFlagsToCmd(cmd, &globalNetworkFlags, false, networkoptions.DefaultSupportedNetworkOptions) - cmd.Flags().BoolVar(&useLatestAvalanchegoReleaseVersion, "latest-avalanchego-version", false, "install latest avalanchego release version on node/s") - cmd.Flags().BoolVar(&useLatestAvalanchegoPreReleaseVersion, "latest-avalanchego-pre-release-version", true, "install latest avalanchego pre-release version on node/s") + cmd.Flags().BoolVar(&latestAvagoReleaseVersion, "latest-avalanchego-version", true, "install latest avalanchego release version on node/s") + cmd.Flags().BoolVar(&latestAvagoPreReleaseVersion, "latest-avalanchego-pre-release-version", false, "install latest avalanchego pre-release version on node/s") cmd.Flags().StringVar(&useCustomAvalanchegoVersion, "custom-avalanchego-version", "", "install given avalanchego version on node/s") cmd.Flags().StringVar(&avalanchegoBinaryPath, "avalanchego-path", "", "use this avalanchego binary path") cmd.Flags().StringArrayVar(&bootstrapIDs, "bootstrap-id", []string{}, "nodeIDs of bootstrap nodes") @@ -140,8 +142,8 @@ func newLocalTrackCmd() *cobra.Command { RunE: localTrack, } cmd.Flags().StringVar(&avalanchegoBinaryPath, "avalanchego-path", "", "use this avalanchego binary path") - cmd.Flags().BoolVar(&useLatestAvalanchegoReleaseVersion, "latest-avalanchego-version", false, "install latest avalanchego release version on node/s") - cmd.Flags().BoolVar(&useLatestAvalanchegoPreReleaseVersion, "latest-avalanchego-pre-release-version", true, "install latest avalanchego pre-release version on node/s") + cmd.Flags().BoolVar(&latestAvagoReleaseVersion, "latest-avalanchego-version", true, "install latest avalanchego release version on node/s") + cmd.Flags().BoolVar(&latestAvagoPreReleaseVersion, "latest-avalanchego-pre-release-version", false, "install latest avalanchego pre-release version on node/s") cmd.Flags().StringVar(&useCustomAvalanchegoVersion, "custom-avalanchego-version", "", "install given avalanchego version on node/s") return cmd } @@ -183,13 +185,13 @@ func localStartNode(_ *cobra.Command, args []string) error { StakingTLSKeyPath: stakingTLSKeyPath, } if useCustomAvalanchegoVersion != "" { - useLatestAvalanchegoReleaseVersion = false - useLatestAvalanchegoPreReleaseVersion = false + latestAvagoPreReleaseVersion = false + latestAvagoReleaseVersion = false } avaGoVersionSetting := node.AvalancheGoVersionSettings{ UseCustomAvalanchegoVersion: useCustomAvalanchegoVersion, - UseLatestAvalanchegoPreReleaseVersion: useLatestAvalanchegoPreReleaseVersion, - UseLatestAvalanchegoReleaseVersion: useLatestAvalanchegoReleaseVersion, + UseLatestAvalanchegoPreReleaseVersion: latestAvagoPreReleaseVersion, + UseLatestAvalanchegoReleaseVersion: latestAvagoReleaseVersion, } nodeConfig := make(map[string]interface{}) if nodeConfigPath != "" { @@ -228,13 +230,13 @@ func localDestroyNode(_ *cobra.Command, args []string) error { func localTrack(_ *cobra.Command, args []string) error { if avalanchegoBinaryPath == "" { if useCustomAvalanchegoVersion != "" { - useLatestAvalanchegoReleaseVersion = false - useLatestAvalanchegoPreReleaseVersion = false + latestAvagoReleaseVersion = false + latestAvagoPreReleaseVersion = false } avaGoVersionSetting := node.AvalancheGoVersionSettings{ UseCustomAvalanchegoVersion: useCustomAvalanchegoVersion, - UseLatestAvalanchegoPreReleaseVersion: useLatestAvalanchegoPreReleaseVersion, - UseLatestAvalanchegoReleaseVersion: useLatestAvalanchegoReleaseVersion, + UseLatestAvalanchegoPreReleaseVersion: latestAvagoPreReleaseVersion, + UseLatestAvalanchegoReleaseVersion: latestAvagoReleaseVersion, } avalancheGoVersion, err := node.GetAvalancheGoVersion(app, avaGoVersionSetting) if err != nil { diff --git a/cmd/nodecmd/node.go b/cmd/nodecmd/node.go index e438470e8..82001b135 100644 --- a/cmd/nodecmd/node.go +++ b/cmd/nodecmd/node.go @@ -60,5 +60,7 @@ rest of the commands to maintain your node and make your node a Subnet Validator cmd.AddCommand(newImportCmd()) // node local cmd.AddCommand(newLocalCmd()) + // node setup + cmd.AddCommand(newSetupCmd()) return cmd } diff --git a/cmd/nodecmd/setup.go b/cmd/nodecmd/setup.go new file mode 100644 index 000000000..f0f861680 --- /dev/null +++ b/cmd/nodecmd/setup.go @@ -0,0 +1,233 @@ +// Copyright (C) 2022, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. +package nodecmd + +import ( + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/ava-labs/avalanche-cli/pkg/prompts" + + "github.com/ava-labs/avalanche-cli/pkg/node" + + "github.com/ava-labs/avalanche-cli/pkg/docker" + + "github.com/ava-labs/avalanche-cli/pkg/cobrautils" + "github.com/ava-labs/avalanche-cli/pkg/constants" + "github.com/ava-labs/avalanche-cli/pkg/models" + "github.com/ava-labs/avalanche-cli/pkg/networkoptions" + "github.com/ava-labs/avalanche-cli/pkg/ssh" + "github.com/ava-labs/avalanche-cli/pkg/utils" + "github.com/ava-labs/avalanche-cli/pkg/ux" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/spf13/cobra" +) + +var ( + nodeIPs []string + sshKeyPaths []string + overrideExisting bool +) + +func newSetupCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "setup", + Short: "Sets up a new Avalanche Node on remote server", + Long: `The node setup command installs Avalanche Go on specified remote servers. +To run the command, the remote servers' IP addresses and SSH private keys are required. + +Currently, only Ubuntu operating system is supported.`, + Args: cobrautils.ExactArgs(0), + RunE: setupNode, + PersistentPostRun: handlePostRun, + } + networkoptions.AddNetworkFlagsToCmd(cmd, &globalNetworkFlags, false, networkoptions.NonLocalSupportedNetworkOptions) + cmd.Flags().BoolVar(&useLatestAvalanchegoReleaseVersion, "latest-avalanchego-version", false, "install latest avalanchego release version on node/s") + cmd.Flags().BoolVar(&useLatestAvalanchegoPreReleaseVersion, "latest-avalanchego-pre-release-version", false, "install latest avalanchego pre-release version on node/s") + cmd.Flags().StringVar(&useCustomAvalanchegoVersion, "custom-avalanchego-version", "", "install given avalanchego version on node/s") + cmd.Flags().StringVar(&useAvalanchegoVersionFromSubnet, "avalanchego-version-from-subnet", "", "install latest avalanchego version, that is compatible with the given subnet, on node/s") + cmd.Flags().BoolVar(&publicHTTPPortAccess, "public-http-port", false, "allow public access to avalanchego HTTP port") + cmd.Flags().StringArrayVar(&nodeIPs, "node-ips", []string{}, "IP addresses of nodes") + cmd.Flags().StringArrayVar(&sshKeyPaths, "ssh-key-paths", []string{}, "ssh key paths") + cmd.Flags().BoolVar(&useSSHAgent, "use-ssh-agent", false, "use ssh agent(ex: Yubikey) for ssh auth") + cmd.Flags().StringVar(&genesisPath, "genesis", "", "path to genesis file") + cmd.Flags().StringVar(&upgradePath, "upgrade", "", "path to upgrade file") + cmd.Flags().BoolVar(&partialSync, "partial-sync", true, "primary network partial sync") + cmd.Flags().BoolVar(&overrideExisting, "override-existing", false, "override existing staking files") + return cmd +} + +func setup(hosts []*models.Host, avalancheGoVersion string, network models.Network) error { + if globalNetworkFlags.UseDevnet { + partialSync = false + ux.Logger.PrintToUser("disabling partial sync default for devnet") + } + ux.Logger.PrintToUser("Setting up Avalanche node(s)...") + wg := sync.WaitGroup{} + wgResults := models.NodeResults{} + spinSession := ux.NewUserSpinner() + + for _, host := range hosts { + wg.Add(1) + go func(nodeResults *models.NodeResults, host *models.Host) { + defer wg.Done() + if err := host.Connect(0); err != nil { + nodeResults.AddResult(host.IP, nil, err) + return + } + if err := provideStakingCertAndKey(host); err != nil { + nodeResults.AddResult(host.IP, nil, err) + return + } + spinner := spinSession.SpinToUser(utils.ScriptLog(host.IP, "Setup Node")) + if err := ssh.RunSSHSetupNode(host, app.Conf.GetConfigPath()); err != nil { + nodeResults.AddResult(host.IP, nil, err) + ux.SpinFailWithError(spinner, "", err) + return + } + if err := ssh.RunSSHSetupDockerService(host); err != nil { + nodeResults.AddResult(host.IP, nil, err) + ux.SpinFailWithError(spinner, "", err) + return + } + ux.SpinComplete(spinner) + spinner = spinSession.SpinToUser(utils.ScriptLog(host.IP, "Setup AvalancheGo")) + // check if host is a API host + if err := docker.ComposeSSHSetupNode(host, + network, + avalancheGoVersion, + bootstrapIDs, + bootstrapIPs, + partialSync, + genesisPath, + upgradePath, + addMonitoring, + host.APINode); err != nil { + nodeResults.AddResult(host.IP, nil, err) + ux.SpinFailWithError(spinner, "", err) + return + } + ux.SpinComplete(spinner) + }(&wgResults, host) + } + wg.Wait() + spinSession.Stop() + for _, node := range hosts { + if wgResults.HasIDWithError(node.NodeID) { + ux.Logger.RedXToUser("Node %s has ERROR: %s", node.IP, wgResults.GetErrorHostMap()[node.IP]) + } + } + + if wgResults.HasErrors() { + return fmt.Errorf("failed to deploy node(s) %s", wgResults.GetErrorHostMap()) + } else { + ux.Logger.PrintToUser(logging.Green.Wrap("AvalancheGo and Avalanche-CLI installed and node(s) are bootstrapping!")) + } + return nil +} + +func setupNode(_ *cobra.Command, _ []string) error { + network, err := networkoptions.GetNetworkFromCmdLineFlags( + app, + "", + globalNetworkFlags, + false, + true, + networkoptions.NonLocalSupportedNetworkOptions, + "", + ) + if err != nil { + return err + } + avaGoVersionSetting := node.AvalancheGoVersionSettings{ + UseAvalanchegoVersionFromSubnet: useAvalanchegoVersionFromSubnet, + UseLatestAvalanchegoReleaseVersion: useLatestAvalanchegoReleaseVersion, + UseLatestAvalanchegoPreReleaseVersion: useLatestAvalanchegoPreReleaseVersion, + UseCustomAvalanchegoVersion: useCustomAvalanchegoVersion, + } + avalancheGoVersion, err := node.GetAvalancheGoVersion(app, avaGoVersionSetting) + if err != nil { + return err + } + + if !useSSHAgent { + if len(nodeIPs) != len(sshKeyPaths) { + return fmt.Errorf("--node-ips and --ssh-key-paths should have same number of values") + } + } + + if err = promptSetupNodes(); err != nil { + return err + } + + hosts := []*models.Host{} + for i, nodeIP := range nodeIPs { + sshKeyPath := "" + if !useSSHAgent { + sshKeyPath = sshKeyPaths[i] + } + hosts = append(hosts, &models.Host{ + SSHUser: constants.RemoteSSHUser, + IP: nodeIP, + SSHPrivateKeyPath: sshKeyPath, + }) + } + if err = setup(hosts, avalancheGoVersion, network); err != nil { + return err + } + printSetupResults(hosts) + return nil +} + +func printSetupResults(hosts []*models.Host) { + for _, host := range hosts { + nodePath := app.GetNodeStakingDir(host.IP) + certBytes, err := os.ReadFile(filepath.Join(nodePath, constants.StakerCertFileName)) + if err != nil { + continue + } + nodeID, err := utils.ToNodeID(certBytes) + if err != nil { + continue + } + ux.Logger.PrintToUser("%s Public IP: %s | %s ", logging.Green.Wrap(">"), host.IP, logging.Green.Wrap(nodeID.String())) + ux.Logger.PrintToUser("staker.crt, staker.key and signer.key are stored at %s. Please keep them safe, as these files can be used to fully recreate your node.", nodePath) + ux.Logger.PrintLineSeparator() + } +} + +func promptSetupNodes() error { + var err error + var numNodes int + ux.Logger.PrintToUser("Only Ubuntu operating system is supported") + if len(nodeIPs) == 0 && len(sshKeyPaths) == 0 { + numNodes, err = app.Prompt.CaptureInt( + "How many Avalanche nodes do you want to setup?", + prompts.ValidatePositiveInt, + ) + } + if err != nil { + return err + } + for len(nodeIPs) < numNodes { + ux.Logger.PrintToUser("Getting info for node %d", len(nodeIPs)+1) + ipAddress, err := app.Prompt.CaptureString("What is the IP address of the node to be set up?") + if err != nil { + return err + } + nodeIPs = append(nodeIPs, ipAddress) + ux.Logger.GreenCheckmarkToUser("Node %d:", len(nodeIPs)) + ux.Logger.PrintToUser("- IP Address: %s", ipAddress) + if !useSSHAgent { + sshKeyPath, err := app.Prompt.CaptureString("What is the key path of the private key that can be used to ssh into this node?") + if err != nil { + return err + } + sshKeyPaths = append(sshKeyPaths, sshKeyPath) + ux.Logger.PrintToUser("- SSH Key Path: %s", sshKeyPath) + } + } + return nil +} diff --git a/cmd/nodecmd/wiz.go b/cmd/nodecmd/wiz.go index 33bac722f..28d6e6b3a 100644 --- a/cmd/nodecmd/wiz.go +++ b/cmd/nodecmd/wiz.go @@ -958,7 +958,7 @@ func setUpSubnetLogging(clusterName, subnetName string) error { } wg.Wait() for _, node := range hosts { - if wgResults.HasNodeIDWithError(node.NodeID) { + if wgResults.HasIDWithError(node.NodeID) { ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID]) } } diff --git a/pkg/application/app.go b/pkg/application/app.go index 0d175a96a..268ccd556 100644 --- a/pkg/application/app.go +++ b/pkg/application/app.go @@ -222,6 +222,10 @@ func (app *Avalanche) GetNodeInstanceDirPath(nodeName string) string { return filepath.Join(app.GetNodesDir(), nodeName) } +func (app *Avalanche) GetNodeStakingDir(nodeIP string) string { + return filepath.Join(app.GetNodesDir(), constants.StakingDir, nodeIP) +} + func (app *Avalanche) GetNodeInstanceAvaGoConfigDirPath(nodeName string) string { return filepath.Join(app.GetAnsibleDir(), nodeName) } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 2ea3ee098..8b63f0ee0 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -39,6 +39,7 @@ const ( NodePrometheusConfigFileName = "prometheus.yml" NodeCloudConfigFileName = "node_cloud_config.json" AnsibleDir = "ansible" + StakingDir = "staking" AnsibleHostInventoryFileName = "hosts" ClustersConfigFileName = "cluster_config.json" ClustersConfigVersion = "1" @@ -174,7 +175,7 @@ const ( GCPCloudService = "Google Cloud Platform" AWSDefaultInstanceType = "c5.2xlarge" GCPDefaultInstanceType = "e2-standard-8" - AnsibleSSHUser = "ubuntu" + RemoteSSHUser = "ubuntu" AWSNodeAnsiblePrefix = "aws_node" GCPNodeAnsiblePrefix = "gcp_node" CustomVMDir = "vms" diff --git a/pkg/models/host.go b/pkg/models/host.go index dccf6238e..8e50b0407 100644 --- a/pkg/models/host.go +++ b/pkg/models/host.go @@ -35,6 +35,7 @@ type Host struct { SSHPrivateKeyPath string SSHCommonArgs string Connection *goph.Client + APINode bool } func NewHostConnection(h *Host, port uint) (*goph.Client, error) { diff --git a/pkg/models/host_test.go b/pkg/models/host_test.go index 922d9f2da..b0fc244ba 100644 --- a/pkg/models/host_test.go +++ b/pkg/models/host_test.go @@ -148,14 +148,14 @@ func hostRunTest(t *testing.T) { NodeID: constants.E2EDocker + "_unittest", IP: localhost, SSHPrivateKeyPath: privKey.Name(), - SSHUser: constants.AnsibleSSHUser, + SSHUser: constants.RemoteSSHUser, SSHCommonArgs: constants.AnsibleSSHUseAgentParams, } brokenHost := &Host{ NodeID: constants.E2EDocker + "_broken", IP: localhost, SSHPrivateKeyPath: brokenKey.Name(), - SSHUser: constants.AnsibleSSHUser, + SSHUser: constants.RemoteSSHUser, SSHCommonArgs: constants.AnsibleSSHUseAgentParams, } // good connection diff --git a/pkg/models/result.go b/pkg/models/result.go index 5d50b7870..9e8861e32 100644 --- a/pkg/models/result.go +++ b/pkg/models/result.go @@ -68,11 +68,11 @@ func (nr *NodeResults) GetErrorHostMap() map[string]error { return hostErrors } -func (nr *NodeResults) HasNodeIDWithError(nodeID string) bool { +func (nr *NodeResults) HasIDWithError(id string) bool { nr.Lock.Lock() defer nr.Lock.Unlock() for _, node := range nr.Results { - if node.NodeID == nodeID && node.Err != nil { + if node.NodeID == id && node.Err != nil { return true } } diff --git a/pkg/utils/ssh.go b/pkg/utils/ssh.go index 18d0a6af5..880c354b6 100644 --- a/pkg/utils/ssh.go +++ b/pkg/utils/ssh.go @@ -20,7 +20,7 @@ func GetSSHConnectionString(publicIP, certFilePath string) string { if certFilePath != "" { certFilePath = fmt.Sprintf("-i %s", certFilePath) } - return fmt.Sprintf("ssh %s %s@%s %s", constants.AnsibleSSHShellParams, constants.AnsibleSSHUser, publicIP, certFilePath) + return fmt.Sprintf("ssh %s %s@%s %s", constants.AnsibleSSHShellParams, constants.RemoteSSHUser, publicIP, certFilePath) } // GetSCPTargetPath returns the target path for the given source path and target directory. @@ -28,7 +28,7 @@ func GetSCPTargetPath(ip, path string) string { if ip == "" { return path } - return fmt.Sprintf("%s@%s:%s", constants.AnsibleSSHUser, ip, path) + return fmt.Sprintf("%s@%s:%s", constants.RemoteSSHUser, ip, path) } // GetSCPCommandString returns the SCP command string for the given source and destination paths. diff --git a/tests/e2e/testcases/node/create/suite.go b/tests/e2e/testcases/node/create/suite.go index 795d9c1c5..6a7b6bdad 100644 --- a/tests/e2e/testcases/node/create/suite.go +++ b/tests/e2e/testcases/node/create/suite.go @@ -17,6 +17,7 @@ import ( "github.com/ava-labs/avalanche-cli/pkg/models" "github.com/ava-labs/avalanche-cli/pkg/utils" "github.com/ava-labs/avalanche-cli/tests/e2e/commands" + e2eUtils "github.com/ava-labs/avalanche-cli/tests/e2e/utils" ginkgo "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) @@ -46,7 +47,9 @@ var _ = ginkgo.Describe("[Node create]", func() { re := regexp.MustCompile(`Generated staking keys for host (\S+)\[NodeID-(\S+)\]`) match := re.FindStringSubmatch(output) if len(match) >= 3 { - hostName = match[1] + var err error + hostName, err = e2eUtils.GetE2EHostInstanceID() + gomega.Expect(err).Should(gomega.BeNil()) NodeID = fmt.Sprintf("NodeID-%s", match[2]) } else { ginkgo.Fail("failed to parse hostName and NodeID") diff --git a/tests/e2e/utils/helpers.go b/tests/e2e/utils/helpers.go index ed00364a0..5acc00a85 100644 --- a/tests/e2e/utils/helpers.go +++ b/tests/e2e/utils/helpers.go @@ -20,6 +20,8 @@ import ( "strings" "time" + "github.com/ava-labs/avalanche-cli/pkg/ansible" + "github.com/ava-labs/avalanche-cli/pkg/binutils" "github.com/ava-labs/avalanche-cli/pkg/constants" "github.com/ava-labs/avalanche-cli/pkg/key" @@ -1101,3 +1103,12 @@ func GetKeyTransferFee(output string) (uint64, error) { func GetAPILargeContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), constants.APIRequestLargeTimeout) } + +func GetE2EHostInstanceID() (string, error) { + hosts, err := ansible.GetInventoryFromAnsibleInventoryFile(path.Join(GetBaseDir(), constants.NodesDir, constants.AnsibleInventoryDir, constants.E2EClusterName)) + if err != nil { + return "", err + } + _, cloudHostID, _ := models.HostAnsibleIDToCloudID(hosts[0].NodeID) + return cloudHostID, nil +}