Skip to content

Commit

Permalink
Refactor node setup out of node create (#2522)
Browse files Browse the repository at this point in the history 
* refactor node setup

* fix lint

* address comments

* fix lint

* address comments

* address comments

* address comment

* fix merge

---------

Signed-off-by: sukantoraymond <rsukanto@umich.edu>
  • Loading branch information
@sukantoraymond
sukantoraymond authored Feb 6, 2025
1 parent d32dda3 commit 2c76e3f
Show file tree
Hide file tree
Showing 14 changed files with 373 additions and 120 deletions.
148 changes: 72 additions & 76 deletions cmd/nodecmd/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/ava-labs/avalanche-cli/pkg/utils"
"github.com/ava-labs/avalanche-cli/pkg/ux"
"github.com/ava-labs/avalanche-cli/pkg/vm"
sdkUtils "github.com/ava-labs/avalanche-cli/sdk/utils"
"github.com/ava-labs/avalanchego/ids"
"github.com/ava-labs/avalanchego/staking"
"github.com/ava-labs/avalanchego/utils/logging"
Expand Down Expand Up @@ -661,7 +662,6 @@ func createNodes(cmd *cobra.Command, args []string) error {
}
return fmt.Errorf("failed to provision node(s) %s", failedHosts.GetNodeList())
}
ux.Logger.PrintToUser("Starting bootstrap process on the newly created Avalanche node(s)...")
wg := sync.WaitGroup{}
wgResults := models.NodeResults{}
spinSession := ux.NewUserSpinner()
Expand All @@ -672,6 +672,7 @@ func createNodes(cmd *cobra.Command, args []string) error {
}
startTime := time.Now()
if addMonitoring {
spinSession := ux.NewUserSpinner()
if len(monitoringHosts) != 1 {
return fmt.Errorf("expected only one monitoring host, found %d", len(monitoringHosts))
}
Expand All @@ -682,116 +683,91 @@ func createNodes(cmd *cobra.Command, args []string) error {
go func(nodeResults *models.NodeResults, monitoringHost *models.Host) {
defer wg.Done()
if err := monitoringHost.Connect(0); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
return
}
spinner := spinSession.SpinToUser(utils.ScriptLog(monitoringHost.NodeID, "Setup Monitoring"))
spinner := spinSession.SpinToUser(utils.ScriptLog(monitoringHost.IP, "Setup Monitoring"))
if err = app.SetupMonitoringEnv(); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
if err = ssh.RunSSHSetupDockerService(monitoringHost); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.Logger.Info("SetupMonitoringEnv RunSSHSetupDockerService completed")
if err = ssh.RunSSHSetupMonitoringFolders(monitoringHost); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.Logger.Info("RunSSHSetupMonitoringFolders completed")
if err := ssh.RunSSHCopyMonitoringDashboards(monitoringHost, app.GetMonitoringDashboardDir()+"/"); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.Logger.Info("RunSSHCopyMonitoringDashboards completed")
if err := ssh.RunSSHSetupPrometheusConfig(monitoringHost, avalancheGoPorts, machinePorts, ltPorts); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.Logger.Info("RunSSHSetupPrometheusConfig completed")
if err := ssh.RunSSHSetupLokiConfig(monitoringHost, constants.AvalancheGoLokiPort); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.Logger.Info("RunSSHSetupLokiConfig completed")
if err := docker.ComposeSSHSetupMonitoring(monitoringHost); err != nil {
nodeResults.AddResult(monitoringHost.NodeID, nil, err)
nodeResults.AddResult(monitoringHost.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.Logger.Info("ComposeSSHSetupMonitoring completed")
ux.SpinComplete(spinner)
}(&wgResults, monitoringHost)
}
wg.Wait()
spinSession.Stop()
}
for _, host := range hosts {
wg.Add(1)
go func(nodeResults *models.NodeResults, host *models.Host) {
defer wg.Done()
if err := host.Connect(0); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
if err := provideStakingCertAndKey(host); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
return
}
spinner := spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Setup Node"))
if err := ssh.RunSSHSetupNode(host, app.Conf.GetConfigPath()); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
if err := ssh.RunSSHSetupDockerService(host); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.SpinComplete(spinner)
if addMonitoring {
cloudID := host.GetCloudID()
nodeID, err := getNodeID(app.GetNodeInstanceDirPath(cloudID))
if err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
if err = ssh.RunSSHSetupPromtailConfig(host, monitoringNodeConfig.PublicIPs[0], constants.AvalancheGoLokiPort, cloudID, nodeID.String(), ""); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
publicAccessToHTTPPort := slices.Contains(cloudConfigMap.GetAllAPIInstanceIDs(), host.GetCloudID()) || publicHTTPPortAccess
host.APINode = publicAccessToHTTPPort
}
if err = setup(hosts, avalancheGoVersion, network); err != nil {
return err
}
if addMonitoring {
spinSession := ux.NewUserSpinner()
for _, host := range hosts {
wg.Add(1)
go func(nodeResults *models.NodeResults, host *models.Host) {
defer wg.Done()
spinner := spinSession.SpinToUser(utils.ScriptLog(host.IP, "Add Monitoring"))
if addMonitoring {
cloudID := host.GetCloudID()
nodeID, err := getNodeID(app.GetNodeInstanceDirPath(cloudID))
if err != nil {
nodeResults.AddResult(host.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
if err = ssh.RunSSHSetupPromtailConfig(host, monitoringNodeConfig.PublicIPs[0], constants.AvalancheGoLokiPort, cloudID, nodeID.String(), ""); err != nil {
nodeResults.AddResult(host.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.SpinComplete(spinner)
}
ux.SpinComplete(spinner)
}
spinner = spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Setup AvalancheGo"))
// check if host is a API host
publicAccessToHTTPPort := slices.Contains(cloudConfigMap.GetAllAPIInstanceIDs(), host.GetCloudID()) || publicHTTPPortAccess
if err := docker.ComposeSSHSetupNode(host,
network,
avalancheGoVersion,
bootstrapIDs,
bootstrapIPs,
partialSync,
genesisPath,
upgradePath,
addMonitoring,
publicAccessToHTTPPort,
); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
ux.SpinComplete(spinner)
}(&wgResults, host)
}(&wgResults, host)
}
wg.Wait()
spinSession.Stop()
}
wg.Wait()
ux.Logger.Info("Create and setup nodes time took: %s", time.Since(startTime))
spinSession.Stop()
if network.Kind == models.Devnet {
Expand All @@ -800,8 +776,8 @@ func createNodes(cmd *cobra.Command, args []string) error {
}
}
for _, node := range hosts {
if wgResults.HasNodeIDWithError(node.NodeID) {
ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID])
if wgResults.HasIDWithError(node.IP) {
ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.IP, wgResults.GetErrorHostMap()[node.IP])
}
}

Expand Down Expand Up @@ -1033,18 +1009,38 @@ func generateNodeCertAndKeys(stakerCertFilePath, stakerKeyFilePath, blsKeyFilePa
}

func provideStakingCertAndKey(host *models.Host) error {
instanceID := host.GetCloudID()
keyPath := filepath.Join(app.GetNodesDir(), instanceID)
keyPath := app.GetNodeStakingDir(host.IP)
if sdkUtils.DirExists(keyPath) && !overrideExisting {
yes, err := app.Prompt.CaptureNoYes(fmt.Sprintf("Directory %s alreday exists. Do you want to override it?", keyPath))
if err != nil {
return err
}
if !yes {
return nil
}
}
nodeID, err := generateNodeCertAndKeys(
filepath.Join(keyPath, constants.StakerCertFileName),
filepath.Join(keyPath, constants.StakerKeyFileName),
filepath.Join(keyPath, constants.BLSKeyFileName),
)
if err != nil {
ux.Logger.PrintToUser("Failed to generate staking keys for host %s", instanceID)
ux.Logger.PrintToUser("Failed to generate staking keys for host %s", host.IP)
return err
} else {
ux.Logger.GreenCheckmarkToUser("Generated staking keys for host %s[%s] ", instanceID, nodeID.String())
ux.Logger.GreenCheckmarkToUser("Generated staking keys for host %s[%s] ", host.IP, nodeID.String())
}
instanceID := host.GetCloudID()
if instanceID != "" {
if err := utils.FileCopy(filepath.Join(keyPath, constants.StakerCertFileName), filepath.Join(app.GetNodesDir(), instanceID, constants.StakerCertFileName)); err != nil {
return err
}
if err := utils.FileCopy(filepath.Join(keyPath, constants.StakerKeyFileName), filepath.Join(app.GetNodesDir(), instanceID, constants.StakerKeyFileName)); err != nil {
return err
}
if err := utils.FileCopy(filepath.Join(keyPath, constants.BLSKeyFileName), filepath.Join(app.GetNodesDir(), instanceID, constants.BLSKeyFileName)); err != nil {
return err
}
}
return ssh.RunSSHUploadStakingFiles(host, keyPath)
}
Expand Down Expand Up @@ -1214,9 +1210,9 @@ func waitForHosts(hosts []*models.Host) *models.NodeResults {
createdWaitGroup.Add(1)
go func(nodeResults *models.NodeResults, host *models.Host) {
defer createdWaitGroup.Done()
spinner := spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Waiting for instance response"))
spinner := spinSession.SpinToUser(utils.ScriptLog(host.IP, "Waiting for instance response"))
if err := host.WaitForSSHShell(constants.SSHServerStartTimeout); err != nil {
nodeResults.AddResult(host.NodeID, nil, err)
nodeResults.AddResult(host.IP, nil, err)
ux.SpinFailWithError(spinner, "", err)
return
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/nodecmd/create_devnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func setupDevnet(clusterName string, hosts []*models.Host, apiNodeIPMap map[stri
wg.Wait()
ux.Logger.PrintLineSeparator()
for _, node := range hosts {
if wgResults.HasNodeIDWithError(node.NodeID) {
if wgResults.HasIDWithError(node.NodeID) {
ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID])
} else {
nodeID, err := getNodeID(app.GetNodeInstanceDirPath(node.GetCloudID()))
Expand Down
70 changes: 36 additions & 34 deletions cmd/nodecmd/local.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,28 +41,30 @@ import (
var (
avalanchegoBinaryPath string

bootstrapIDs []string
bootstrapIPs []string
genesisPath string
upgradePath string
stakingTLSKeyPath string
stakingCertKeyPath string
stakingSignerKeyPath string
numNodes uint32
nodeConfigPath string
partialSync bool
stakeAmount uint64
rpcURL string
balanceAVAX float64
remainingBalanceOwnerAddr string
disableOwnerAddr string
aggregatorLogLevel string
aggregatorLogToStdout bool
delegationFee uint16
publicKey string
pop string
minimumStakeDuration uint64
validatorManagerAddress string
bootstrapIDs []string
bootstrapIPs []string
genesisPath string
upgradePath string
stakingTLSKeyPath string
stakingCertKeyPath string
stakingSignerKeyPath string
numNodes uint32
nodeConfigPath string
partialSync bool
stakeAmount uint64
rpcURL string
balanceAVAX float64
remainingBalanceOwnerAddr string
disableOwnerAddr string
aggregatorLogLevel string
aggregatorLogToStdout bool
delegationFee uint16
publicKey string
pop string
minimumStakeDuration uint64
latestAvagoReleaseVersion bool
latestAvagoPreReleaseVersion bool
validatorManagerAddress string
)

// const snapshotName = "local_snapshot"
Expand Down Expand Up @@ -104,8 +106,8 @@ You can check the bootstrapping status by running avalanche node status local.
PersistentPostRun: handlePostRun,
}
networkoptions.AddNetworkFlagsToCmd(cmd, &globalNetworkFlags, false, networkoptions.DefaultSupportedNetworkOptions)
cmd.Flags().BoolVar(&useLatestAvalanchegoReleaseVersion, "latest-avalanchego-version", false, "install latest avalanchego release version on node/s")
cmd.Flags().BoolVar(&useLatestAvalanchegoPreReleaseVersion, "latest-avalanchego-pre-release-version", true, "install latest avalanchego pre-release version on node/s")
cmd.Flags().BoolVar(&latestAvagoReleaseVersion, "latest-avalanchego-version", true, "install latest avalanchego release version on node/s")
cmd.Flags().BoolVar(&latestAvagoPreReleaseVersion, "latest-avalanchego-pre-release-version", false, "install latest avalanchego pre-release version on node/s")
cmd.Flags().StringVar(&useCustomAvalanchegoVersion, "custom-avalanchego-version", "", "install given avalanchego version on node/s")
cmd.Flags().StringVar(&avalanchegoBinaryPath, "avalanchego-path", "", "use this avalanchego binary path")
cmd.Flags().StringArrayVar(&bootstrapIDs, "bootstrap-id", []string{}, "nodeIDs of bootstrap nodes")
Expand Down Expand Up @@ -140,8 +142,8 @@ func newLocalTrackCmd() *cobra.Command {
RunE: localTrack,
}
cmd.Flags().StringVar(&avalanchegoBinaryPath, "avalanchego-path", "", "use this avalanchego binary path")
cmd.Flags().BoolVar(&useLatestAvalanchegoReleaseVersion, "latest-avalanchego-version", false, "install latest avalanchego release version on node/s")
cmd.Flags().BoolVar(&useLatestAvalanchegoPreReleaseVersion, "latest-avalanchego-pre-release-version", true, "install latest avalanchego pre-release version on node/s")
cmd.Flags().BoolVar(&latestAvagoReleaseVersion, "latest-avalanchego-version", true, "install latest avalanchego release version on node/s")
cmd.Flags().BoolVar(&latestAvagoPreReleaseVersion, "latest-avalanchego-pre-release-version", false, "install latest avalanchego pre-release version on node/s")
cmd.Flags().StringVar(&useCustomAvalanchegoVersion, "custom-avalanchego-version", "", "install given avalanchego version on node/s")
return cmd
}
Expand Down Expand Up @@ -183,13 +185,13 @@ func localStartNode(_ *cobra.Command, args []string) error {
StakingTLSKeyPath: stakingTLSKeyPath,
}
if useCustomAvalanchegoVersion != "" {
useLatestAvalanchegoReleaseVersion = false
useLatestAvalanchegoPreReleaseVersion = false
latestAvagoPreReleaseVersion = false
latestAvagoReleaseVersion = false
}
avaGoVersionSetting := node.AvalancheGoVersionSettings{
UseCustomAvalanchegoVersion: useCustomAvalanchegoVersion,
UseLatestAvalanchegoPreReleaseVersion: useLatestAvalanchegoPreReleaseVersion,
UseLatestAvalanchegoReleaseVersion: useLatestAvalanchegoReleaseVersion,
UseLatestAvalanchegoPreReleaseVersion: latestAvagoPreReleaseVersion,
UseLatestAvalanchegoReleaseVersion: latestAvagoReleaseVersion,
}
nodeConfig := make(map[string]interface{})
if nodeConfigPath != "" {
Expand Down Expand Up @@ -228,13 +230,13 @@ func localDestroyNode(_ *cobra.Command, args []string) error {
func localTrack(_ *cobra.Command, args []string) error {
if avalanchegoBinaryPath == "" {
if useCustomAvalanchegoVersion != "" {
useLatestAvalanchegoReleaseVersion = false
useLatestAvalanchegoPreReleaseVersion = false
latestAvagoReleaseVersion = false
latestAvagoPreReleaseVersion = false
}
avaGoVersionSetting := node.AvalancheGoVersionSettings{
UseCustomAvalanchegoVersion: useCustomAvalanchegoVersion,
UseLatestAvalanchegoPreReleaseVersion: useLatestAvalanchegoPreReleaseVersion,
UseLatestAvalanchegoReleaseVersion: useLatestAvalanchegoReleaseVersion,
UseLatestAvalanchegoPreReleaseVersion: latestAvagoPreReleaseVersion,
UseLatestAvalanchegoReleaseVersion: latestAvagoReleaseVersion,
}
avalancheGoVersion, err := node.GetAvalancheGoVersion(app, avaGoVersionSetting)
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions cmd/nodecmd/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,7 @@ rest of the commands to maintain your node and make your node a Subnet Validator
cmd.AddCommand(newImportCmd())
// node local
cmd.AddCommand(newLocalCmd())
// node setup
cmd.AddCommand(newSetupCmd())
return cmd
}
Loading

0 comments on commit 2c76e3f

Please sign in to comment.