Set debug level in system tests for agent (#1660)

Set debug log level in system tests for agent, once system tests
are finished the log level is set with the previous value.

internal/kibana/agents.go

@@ -29,6 +29,11 @@ type Agent struct {
 		Host struct {
 			Name string `json:"name"`
 		} `json:"host"`
+		Elastic struct {
+			Agent struct {
+				LogLevel string `json:"log_level"`
+			} `json:"agent"`
+		} `json:"elastic"`
 	} `json:"local_metadata"`
 }
 

internal/kibana/fleet.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 )
 
 // DefaultFleetServerURL returns the default Fleet server configured in Kibana
@@ -43,3 +44,52 @@ func (c *Client) DefaultFleetServerURL() (string, error) {
 
 	return "", errors.New("could not find the fleet server URL for this project")
 }
+
+func (c *Client) SetAgentLogLevel(agentID, level string) error {
+	path := fmt.Sprintf("%s/agents/%s/actions", FleetAPI, agentID)
+
+	type fleetAction struct {
+		Action struct {
+			Type string `json:"type"`
+			Data struct {
+				LogLevel string `json:"log_level"`
+			} `json:"data"`
+		} `json:"action"`
+	}
+
+	action := fleetAction{}
+	action.Action.Type = "SETTINGS"
+	action.Action.Data.LogLevel = level
+
+	reqBody, err := json.Marshal(action)
+	if err != nil {
+		return fmt.Errorf("could not convert action settingr (request) to JSON: %w", err)
+	}
+
+	statusCode, respBody, err := c.post(path, reqBody)
+	if err != nil {
+		return fmt.Errorf("could not update agent settings: %w", err)
+	}
+
+	if statusCode != http.StatusOK {
+		return fmt.Errorf("could not set new log level; API status code = %d; response body = %s", statusCode, respBody)
+	}
+
+	type actionResponse struct {
+		ID        string    `json:"id"`
+		CreatedAt time.Time `json:"created_at"`
+		Type      string    `json:"type"`
+		Data      struct {
+			LogLevel string `json:"log_level"`
+		} `json:"data"`
+		Agents []string `json:"agents"`
+	}
+	var resp struct {
+		Item actionResponse `json:"item"`
+	}
+
+	if err := json.Unmarshal(respBody, &resp); err != nil {
+		return fmt.Errorf("could not convert actions agent (response) to JSON: %w", err)
+	}
+	return nil
+}

internal/testrunner/runners/system/runner.go

@@ -102,11 +102,12 @@ type runner struct {
 	options   testrunner.TestOptions
 	pipelines []ingest.Pipeline
 	// Execution order of following handlers is defined in runner.TearDown() method.
-	deleteTestPolicyHandler func() error
-	deletePackageHandler    func() error
-	resetAgentPolicyHandler func() error
-	shutdownServiceHandler  func() error
-	wipeDataStreamHandler   func() error
+	deleteTestPolicyHandler   func() error
+	deletePackageHandler      func() error
+	resetAgentPolicyHandler   func() error
+	resetAgentLogLevelHandler func() error
+	shutdownServiceHandler    func() error
+	wipeDataStreamHandler     func() error
 }
 
 // Type returns the type of test that can be run by this test runner.
@@ -153,6 +154,13 @@ func (r *runner) tearDownTest() error {
 		r.resetAgentPolicyHandler = nil
 	}
 
+	if r.resetAgentLogLevelHandler != nil {
+		if err := r.resetAgentLogLevelHandler(); err != nil {
+			return err
+		}
+		r.resetAgentLogLevelHandler = nil
+	}
+
 	if r.deleteTestPolicyHandler != nil {
 		if err := r.deleteTestPolicyHandler(); err != nil {
 			return err
@@ -632,6 +640,21 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 		Revision: agent.PolicyRevision,
 	}
 
+	logger.Debug("Set Debug log level to agent")
+	origLogLevel := agent.LocalMetadata.Elastic.Agent.LogLevel
+	err = r.options.KibanaClient.SetAgentLogLevel(agent.ID, "debug")
+	if err != nil {
+		return result.WithError(fmt.Errorf("error setting log level debug for agent %s: %w", agent.ID, err))
+	}
+	r.resetAgentLogLevelHandler = func() error {
+		logger.Debugf("reassigning original log level %q back to agent...", origLogLevel)
+
+		if err := r.options.KibanaClient.SetAgentLogLevel(agent.ID, origLogLevel); err != nil {
+			return fmt.Errorf("error reassigning original log level to agent: %w", err)
+		}
+		return nil
+	}
+
 	// Assign policy to agent
 	r.resetAgentPolicyHandler = func() error {
 		logger.Debug("reassigning original policy back to agent...")