Merge pull request #16 from snonux/develop

Multiple minor enhancements.

cmd/dcat/main.go

@@ -29,6 +29,7 @@ func main() {
 	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
 	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
 	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
 	flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method")

cmd/dgrep/main.go

@@ -31,6 +31,7 @@ func main() {
 	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
 	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
 	flag.IntVar(&sshPort, "port", 2222, "SSH server port")
 	flag.StringVar(&args.Discovery, "discovery", "", "Server discovery method")

cmd/dmap/main.go

@@ -34,6 +34,7 @@ func main() {
 	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
 	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
 	flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection")
 	flag.IntVar(&sshPort, "port", 2222, "SSH server port")

cmd/dtail/main.go

@@ -42,6 +42,7 @@ func main() {
 	flag.BoolVar(&debugEnable, "debug", false, "Activate debug messages")
 	flag.BoolVar(&displayVersion, "version", false, "Display version")
 	flag.BoolVar(&noColor, "noColor", false, "Disable ANSII terminal colors")
+	flag.BoolVar(&args.Quiet, "quiet", false, "Quiet output mode")
 	flag.IntVar(&args.ConnectionsPerCPU, "cpc", 10, "How many connections established per CPU core concurrently")
 	flag.IntVar(&args.Timeout, "timeout", 0, "Max time dtail server will collect data until disconnection")
 	flag.IntVar(&pprof, "pprof", -1, "Start PProf server this port")

go.mod

@@ -5,7 +5,7 @@ go 1.15
 require (
 	github.com/DataDog/zstd v1.4.5
 	golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
+	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
 	golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d // indirect
 	golang.org/x/term v0.0.0-20201207232118-ee85cb95a76b // indirect
 )

go.sum

@@ -14,6 +14,8 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

internal/clients/args.go

@@ -22,4 +22,5 @@ type Args struct {
 	SSHAuthMethods     []gossh.AuthMethod
 	SSHHostKeyCallback gossh.HostKeyCallback
 	PrivateKeyPathFile string
+	Quiet            bool
 }

internal/clients/baseclient.go

@@ -39,7 +39,7 @@ type baseClient struct {
 }
 
 func (c *baseClient) init() {
-	logger.Info("Initiating base client")
+	logger.Debug("Initiating base client")
 
 	flag := regex.Default
 	if c.Args.RegexInvert {
@@ -70,7 +70,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i
 	// Periodically check for unknown hosts, and ask the user whether to trust them or not.
 	go c.hostKeyCallback.PromptAddHosts(ctx)
 	// Print client stats every time something on statsCh is recieved.
-	go c.stats.Start(ctx, c.throttleCh, statsCh)
+	go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet)
 	// Keep count of active connections
 	active := make(chan struct{}, len(c.connections))
 
@@ -127,7 +127,7 @@ func (c *baseClient) makeConnection(server string, sshAuthMethods []gossh.AuthMe
 }
 
 func (c *baseClient) waitUntilDone(ctx context.Context, active chan struct{}) {
-	defer logger.Info("Terminated connection")
+	defer logger.Debug("Terminated connection")
 
 	// We want to have at least one active connection
 	<-active

internal/clients/catclient.go

@@ -42,8 +42,9 @@ func (c CatClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c CatClient) makeCommands() (commands []string) {
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
 	return
 }

internal/clients/grepclient.go

@@ -41,8 +41,10 @@ func (c GrepClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c GrepClient) makeCommands() (commands []string) {
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
+
 	return
 }

internal/clients/handlers/basehandler.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"strconv"
 	"strings"
 	"time"
 
@@ -78,6 +77,7 @@ func (h *baseHandler) Read(p []byte) (n int, err error) {
 	case <-h.Done():
 		return 0, io.EOF
 	}
+
 	return
 }
 
@@ -111,21 +111,5 @@ func (h *baseHandler) handleHiddenMessage(message string) {
 		case <-h.Done():
 			return
 		}
-
-	case strings.HasPrefix(message, ".run exitstatus"):
-		splitted := strings.Split(strings.TrimSuffix(message, "\n"), " ")
-		if len(splitted) != 3 {
-			logger.Error("Unable to retrieve exitstatus", message)
-			return
-		}
-		i, err := strconv.Atoi(splitted[2])
-		if err != nil {
-			logger.Error("Unable to retrieve exitstatus", message, err)
-			return
-		}
-		logger.Debug("Retrieved exitstatus", h.status)
-		if i > h.status {
-			h.status = i
-		}
 	}
 }

internal/clients/handlers/healthhandler.go

@@ -19,6 +19,7 @@ type HealthHandler struct {
 	receive chan<- string
 	// The remote server address
 	server string
+	// The return status.
 	status int
 }
 

internal/clients/maker.go

@@ -4,6 +4,9 @@ import (
 	"github.com/mimecast/dtail/internal/clients/handlers"
 )
 
+// maker interface helps to re-use code in all DTail client implementations.
+// All clients share the baseClient but have different connection handlers
+// and send different commands to the DTail server.
 type maker interface {
 	makeHandler(server string) handlers.Handler
 	makeCommands() (commands []string)

internal/clients/maprclient.go

@@ -99,7 +99,7 @@ func (c *MaprClient) Start(ctx context.Context, statsCh <-chan string) (status i
 
 	status = c.baseClient.Start(ctx, statsCh)
 	if c.cumulative {
-		logger.Info("Received final mapreduce result")
+		logger.Debug("Received final mapreduce result")
 		c.reportResults()
 	}
 
@@ -112,6 +112,7 @@ func (c MaprClient) makeHandler(server string) handlers.Handler {
 
 func (c MaprClient) makeCommands() (commands []string) {
 	commands = append(commands, fmt.Sprintf("map %s", c.query.RawQuery))
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 
 	modeStr := "cat"
 	if c.Mode == omode.TailClient {
@@ -123,7 +124,7 @@ func (c MaprClient) makeCommands() (commands []string) {
 			commands = append(commands, fmt.Sprintf("timeout %d %s %s %s", c.Timeout, modeStr, file, c.Regex.Serialize()))
 			continue
 		}
-		commands = append(commands, fmt.Sprintf("%s %s %s", modeStr, file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", modeStr, options, file, c.Regex.Serialize()))
 	}
 
 	return
@@ -133,7 +134,7 @@ func (c *MaprClient) periodicReportResults(ctx context.Context) {
 	for {
 		select {
 		case <-time.After(c.query.Interval):
-			logger.Info("Gathering interim mapreduce result")
+			logger.Debug("Gathering interim mapreduce result")
 			c.reportResults()
 		case <-ctx.Done():
 			return
@@ -165,7 +166,7 @@ func (c *MaprClient) printResults() {
 	}
 
 	if numLines == 0 {
-		logger.Info("Empty result set this time...")
+		logger.Warn("Empty result set this time...")
 		return
 	}
 

internal/clients/stats.go

@@ -34,7 +34,7 @@ func newTailStats(connectionsTotal int) *stats {
 
 // Start starts printing client connection stats every time a signal is recieved or
 // connection count has changed.
-func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string) {
+func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, quiet bool) {
 	var connectedLast int
 
 	for {
@@ -45,7 +45,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 		case message := <-statsCh:
 			messages = append(messages, message)
 			force = true
-		case <-time.After(time.Second * 10):
+		case <-time.After(time.Second * 3):
 		case <-ctx.Done():
 			return
 		}
@@ -55,15 +55,15 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 
 		newConnections := connected - connectedLast
 
-		if connected == connectedLast && !force {
+		if (connected == connectedLast || quiet) && !force {
 			continue
 		}
 
 		stats := s.statsLine(connected, newConnections, throttle)
 		switch force {
 		case true:
 			messages = append(messages, fmt.Sprintf("Connection stats: %s", stats))
-			s.printStatsOnInterrupt(messages)
+			s.printStatsDueInterrupt(messages)
 		default:
 			logger.Info(stats)
 		}
@@ -75,7 +75,7 @@ func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <
 	}
 }
 
-func (s *stats) printStatsOnInterrupt(messages []string) {
+func (s *stats) printStatsDueInterrupt(messages []string) {
 	logger.Pause()
 	for _, message := range messages {
 		fmt.Println(fmt.Sprintf(" %s", message))
@@ -107,5 +107,6 @@ func percentOf(total float64, value float64) float64 {
 	if total == 0 || total == value {
 		return 100
 	}
+
 	return value / (total / 100.0)
 }

internal/clients/tailclient.go

@@ -29,6 +29,7 @@ func NewTailClient(args Args) (*TailClient, error) {
 
 	c.init()
 	c.makeConnections(c)
+
 	return &c, nil
 }
 
@@ -37,8 +38,9 @@ func (c TailClient) makeHandler(server string) handlers.Handler {
 }
 
 func (c TailClient) makeCommands() (commands []string) {
+	options := fmt.Sprintf("quiet=%v", c.Args.Quiet)
 	for _, file := range strings.Split(c.What, ",") {
-		commands = append(commands, fmt.Sprintf("%s %s %s", c.Mode.String(), file, c.Regex.Serialize()))
+		commands = append(commands, fmt.Sprintf("%s:%s %s %s", c.Mode.String(), options, file, c.Regex.Serialize()))
 	}
 	logger.Debug(commands)
 

internal/color/colorfy.go

@@ -41,16 +41,14 @@ func paintClientStats(line string) string {
 
 // Colorfy a given line based on the line's content.
 func Colorfy(line string) string {
-	if strings.HasPrefix(line, "REMOTE") {
+	switch {
+	case strings.HasPrefix(line, "REMOTE"):
 		return paintRemote(line)
-	}
-	if strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|") {
+	case strings.HasPrefix(line, "CLIENT") && strings.Contains(line, "|stats|"):
 		return paintClientStats(line)
-	}
-	if strings.Contains(line, "ERROR") {
+	case strings.Contains(line, "ERROR"):
 		return Paint(Magenta, line)
-	}
-	if strings.Contains(line, "WARN") {
+	case strings.Contains(line, "WARN"):
 		return Paint(Magenta, line)
 	}