5.1.0 docker networks and rack-ids

CHANGELOG.md

@@ -1,3 +1,9 @@
+#### 5.1.0
+* add option `aerolab config docker` to handle multiple docker networks
+* add option to specify network name when creating docker clusters/clients
+* add interactive option - when a network doesn't exist, ask if one should be created
+* add `aerolab conf rackid` option to add/change rack-id configuration on cluster nodes
+
 #### 5.0.0
 * add `SIGINT` handler to aerospike installer downloader, and template creators, to abort smoothly
 * allow installation of FIPS edition of Aerospike

VERSION.md

@@ -1 +1 @@
-5.0.0
+5.1.0

docs/usage/fullstack_aws.md

@@ -41,12 +41,6 @@ aerolab cluster add exporter -n ${cluster_name}
 aerolab client create ams -n ${ams_name} -s ${cluster_name} -I ${aws_instance_type_ams} -U ${aws_az}
 ```
 
-### Using Docker Desktop without tunneling configured
-
-```
-aerolab client create ams -n ${ams_name} -s ${cluster_name} -e 3000:3000
-```
-
 ## Deploy 5 tools machines for asbenchmark
 
 ```

docs/usage/racks.md

@@ -2,16 +2,28 @@
 AeroLab makes it easy for you to deploy a [rack-aware](https://docs.aerospike.com/server/operations/configure/network/rack-aware)
 Aerospike Database cluster.
 
-## Create a three node Aerospike cluster with rack-id 1
+## Create a size node Aerospike cluster, do not start aerospike
 
 ```bash
-aerolab cluster create -c 3 -o templates/rack1.conf
+aerolab cluster create -c 6 -s n
 ```
 
-## Grow the cluster by another three nodes, using rack-id 2
+## Assign rack-id 1 to first three nodes, all namespaces
 
 ```bash
-aerolab cluster grow -c 3 -o templates/rack2.conf
+aerolab conf rackid -l 1-3 -i 1
+```
+
+## Assign rack-id 2 to the next three nodes, all namespaces
+
+```bash
+aerolab conf rackid -l 4-6 -i 2
+```
+
+## Start aerospike
+
+```bash
+aerolab aerospike start
 ```
 
 ## Confirm the cluster is working

src/backend.go

@@ -24,6 +24,7 @@ type backendExtra struct {
 	exposePorts     []string // docker only
 	switches        string   // docker only
 	dockerHostname  bool     // docker only
+	network         string   // docker only
 	ami             string   // aws only
 	instanceType    string   // aws only
 	ebs             string   // aws only
@@ -116,6 +117,11 @@ type backend interface {
 	ListSecurityGroups() error
 	// may implement
 	ListSubnets() error
+	// may implement (docker related mostly)
+	CreateNetwork(name string, driver string, subnet string, mtu string) error
+	DeleteNetwork(name string) error
+	PruneNetworks() error
+	ListNetworks(csv bool, writer io.Writer) error
 }
 
 // check return code from exec function

src/backendAws.go

@@ -87,6 +87,19 @@ func (d *backendAws) WorkOnServers() {
 	awsTagAerospikeVersion = awsServerTagAerospikeVersion
 }
 
+func (d *backendAws) CreateNetwork(name string, driver string, subnet string, mtu string) error {
+	return nil
+}
+func (d *backendAws) DeleteNetwork(name string) error {
+	return nil
+}
+func (d *backendAws) PruneNetworks() error {
+	return nil
+}
+func (d *backendAws) ListNetworks(csv bool, writer io.Writer) error {
+	return nil
+}
+
 func (d *backendAws) Init() error {
 	var err error
 

src/backendDocker.go

@@ -312,6 +312,75 @@ func (d *backendDocker) TemplateDestroy(v backendVersion) error {
 }
 
 func (d *backendDocker) DeployCluster(v backendVersion, name string, nodeCount int, extra *backendExtra) error {
+	if extra.network != "" {
+		b := new(bytes.Buffer)
+		err := d.ListNetworks(true, b)
+		if err != nil {
+			return err
+		}
+		found := false
+		for i, line := range strings.Split(b.String(), "\n") {
+			if i == 0 {
+				continue
+			}
+			netName := strings.Split(line, ",")[0]
+			if netName == extra.network {
+				found = true
+				break
+			}
+		}
+		if !found {
+			fmt.Printf("Network %s not found! Create (y/n)? ", extra.network)
+			reader := bufio.NewReader(os.Stdin)
+			answer := ""
+			for strings.ToLower(answer) != "y" && strings.ToLower(answer) != "n" && strings.ToLower(answer) != "yes" && strings.ToLower(answer) != "no" {
+				answer, _ = reader.ReadString('\n')
+				answer = strings.Trim(answer, "\t\r\n ")
+				if strings.ToLower(answer) != "y" && strings.ToLower(answer) != "n" && strings.ToLower(answer) != "yes" && strings.ToLower(answer) != "no" {
+					fmt.Println("Invalid input: answer either 'y' or 'n'")
+					fmt.Printf("Network %s not found! Create (y/n)? ", extra.network)
+				}
+			}
+			if strings.HasPrefix(answer, "n") {
+				return fmt.Errorf("Network not found, choose another network or create one first with: aerolab config docker help")
+			}
+			ok := false
+			for !ok {
+				fmt.Printf("Subnet (empty=default): ")
+				subnet, _ := reader.ReadString('\n')
+				subnet = strings.Trim(subnet, "\t\r\n ")
+				fmt.Printf("Driver (empty=default): ")
+				driver, _ := reader.ReadString('\n')
+				driver = strings.Trim(driver, "\t\r\n ")
+				fmt.Printf("MTU (empty=default): ")
+				mtu, _ := reader.ReadString('\n')
+				mtu = strings.Trim(mtu, "\t\r\n ")
+				fmt.Printf("OK (y/n/q)? ")
+				answer := ""
+				for strings.ToLower(answer) != "y" && strings.ToLower(answer) != "n" && strings.ToLower(answer) != "yes" && strings.ToLower(answer) != "no" && strings.ToLower(answer) != "q" && strings.ToLower(answer) != "quit" {
+					answer, _ = reader.ReadString('\n')
+					answer = strings.Trim(answer, "\t\r\n ")
+					if strings.ToLower(answer) != "y" && strings.ToLower(answer) != "n" && strings.ToLower(answer) != "yes" && strings.ToLower(answer) != "no" && strings.ToLower(answer) != "q" && strings.ToLower(answer) != "quit" {
+						fmt.Println("Invalid input: answer either 'y' or 'n'")
+						fmt.Printf("OK (y/n/q)? ")
+					}
+				}
+				if strings.HasPrefix(answer, "q") {
+					return fmt.Errorf("Network not found, choose another network or create one first with: aerolab config docker help")
+				}
+				if strings.HasPrefix(answer, "y") {
+					if driver == "" {
+						driver = "bridge"
+					}
+					err = d.CreateNetwork(extra.network, driver, subnet, mtu)
+					if err != nil {
+						return err
+					}
+					ok = true
+				}
+			}
+		}
+	}
 	if err := d.versionToReal(&v); err != nil {
 		return err
 	}
@@ -374,6 +443,9 @@ func (d *backendDocker) DeployCluster(v backendVersion, name string, nodeCount i
 		if extra.swapLimit != "" {
 			exposeList = append(exposeList, "--memory-swap", extra.swapLimit)
 		}
+		if extra.network != "" {
+			exposeList = append(exposeList, "--network", extra.network)
+		}
 		if extra.privileged {
 			fmt.Println("WARNING: privileged container")
 			exposeList = append(exposeList, "--device-cgroup-rule=b 7:* rmw", "--privileged=true", "--cap-add=NET_ADMIN", "--cap-add=NET_RAW", "-td", "--name", fmt.Sprintf(dockerNameHeader+"%s_%d", name, node), tmplName)

src/backendDockerNetwork.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"text/tabwriter"
+)
+
+type dockerListNetwork struct {
+	Name   string
+	Driver string
+	IPAM   struct {
+		Config []struct {
+			Subnet  string
+			Gateway string
+		}
+	}
+	Options map[string]string
+}
+
+func (d *backendDocker) CreateNetwork(name string, driver string, subnet string, mtu string) error {
+	if driver == "" {
+		driver = "bridge"
+	}
+	opts := []string{"network", "create", "--attachable", "-d", driver}
+	if subnet != "" {
+		opts = append(opts, "--subnet", subnet)
+	}
+	opts = append(opts, "--opt", "com.docker.network.bridge.enable_icc=true", "--opt", "com.docker.network.bridge.enable_ip_masquerade=true", "--opt", "com.docker.network.bridge.host_binding_ipv4=0.0.0.0", "--opt", "com.docker.network.bridge.name="+name)
+	if mtu != "" {
+		opts = append(opts, "--opt", "com.docker.network.driver.mtu="+mtu)
+	}
+	opts = append(opts, name)
+	out, err := exec.Command("docker", opts...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, string(out))
+	}
+	return nil
+}
+
+func (d *backendDocker) DeleteNetwork(name string) error {
+	out, err := exec.Command("docker", "network", "rm", name).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, string(out))
+	}
+	return nil
+}
+
+func (d *backendDocker) PruneNetworks() error {
+	out, err := exec.Command("docker", "network", "prune", "-f").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, string(out))
+	}
+	return nil
+}
+func (d *backendDocker) ListNetworks(csv bool, writer io.Writer) error {
+	out, err := exec.Command("docker", "network", "list", "--format", "{{.Name}}").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, string(out))
+	}
+	networks := []string{"network", "inspect"}
+	for _, line := range strings.Split(string(out), "\n") {
+		line = strings.Trim(line, "\r\t\n ")
+		if line == "" {
+			continue
+		}
+		networks = append(networks, line)
+	}
+	out, err = exec.Command("docker", networks...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, string(out))
+	}
+
+	netlist := []dockerListNetwork{}
+	err = json.Unmarshal(out, &netlist)
+	if err != nil {
+		return err
+	}
+	if writer == nil {
+		writer = os.Stdout
+	}
+	w := tabwriter.NewWriter(writer, 1, 1, 4, ' ', 0)
+	if !csv {
+		fmt.Fprintln(w, "NAME\tDRIVER\tSUBNETS\tMTU")
+		fmt.Fprintln(w, "----\t------\t-------\t---")
+	} else {
+		fmt.Fprintln(writer, "name,driver,subnets,mtu")
+	}
+	for _, net := range netlist {
+		subnets := []string{}
+		for _, sub := range net.IPAM.Config {
+			subnets = append(subnets, sub.Subnet)
+		}
+		mtuOpt, ok := net.Options["com.docker.network.driver.mtu"]
+		if !ok {
+			mtuOpt = "default"
+		}
+		if !csv {
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", net.Name, net.Driver, strings.Join(subnets, ","), mtuOpt)
+		} else {
+			fmt.Fprintf(writer, "%s,%s,%s,%s\n", net.Name, net.Driver, strings.Join(subnets, ","), mtuOpt)
+		}
+	}
+	if !csv {
+		w.Flush()
+	}
+	return nil
+}

src/cmdClientDoBase.go

@@ -111,6 +111,7 @@ func (c *clientCreateBaseCmd) createBase(args []string, nt string) (machines []i
 		ramLimit:        c.Docker.RamLimit,
 		swapLimit:       c.Docker.SwapLimit,
 		privileged:      c.Docker.Privileged,
+		network:         c.Docker.NetworkName,
 		exposePorts:     ep,
 		switches:        c.Docker.ExtraFlags,
 		dockerHostname:  !c.NoSetHostname,

src/cmdClusterCreate.go

@@ -75,6 +75,7 @@ type clusterCreateCmdDocker struct {
 	RamLimit          string `short:"t" long:"ram-limit" description:"Limit RAM available to each node, e.g. 500m, or 1g." default:""`
 	SwapLimit         string `short:"w" long:"swap-limit" description:"Limit the amount of total memory (ram+swap) each node can use, e.g. 600m. If ram-limit==swap-limit, no swap is available." default:""`
 	Privileged        bool   `short:"B" long:"privileged" description:"Docker only: run container in privileged mode"`
+	NetworkName       string `long:"network" description:"specify a network name to use for non-default docker network; for more info see: aerolab config docker help" default:""`
 }
 
 func init() {
@@ -275,6 +276,7 @@ func (c *clusterCreateCmd) realExecute(args []string, isGrow bool) error {
 		ramLimit:        c.Docker.RamLimit,
 		swapLimit:       c.Docker.SwapLimit,
 		privileged:      c.Docker.Privileged,
+		network:         c.Docker.NetworkName,
 		exposePorts:     ep,
 		switches:        c.Docker.ExtraFlags,
 		dockerHostname:  !c.NoSetHostname,

src/cmdConf.go

@@ -5,6 +5,7 @@ import "os"
 type confCmd struct {
 	Generator confGeneratorCmd `command:"generate" subcommands-optional:"true" description:"Generate or modify Aerospike configuration files"`
 	FixMesh   confFixMeshCmd   `command:"fix-mesh" subcommands-optional:"true" description:"Fix mesh configuration in the cluster"`
+	RackID    confRackIdCmd    `command:"rackid" subcommands-optional:"true" description:"Change/add rack-id to namespaces in the existing cluster nodes"`
 	Help      helpCmd          `command:"help" subcommands-optional:"true" description:"Print help"`
 }