Enhanced logs for advanced auth capabilities

treeverse · Aug 4, 2024 · b36ef99 · b36ef99
1 parent 7fdb5c2
commit b36ef99
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 46 deletions.
diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go
@@ -318,6 +318,7 @@ var runCmd = &cobra.Command{
 			cfg.Logging.AuditLogLevel,
 			cfg.Logging.TraceRequestHeaders,
 			cfg.Gateways.S3.VerifyUnsupported,
+			cfg.IsAdvancedAuth(),
 		)
 		s3gatewayHandler = apiAuthenticator(s3gatewayHandler)
 

diff --git a/pkg/api/auth_middleware.go b/pkg/api/auth_middleware.go
@@ -105,7 +105,7 @@ func AuthMiddleware(logger logging.Logger, swagger *openapi3.Swagger, authentica
 				return
 			}
 			if user != nil {
-				ctx := logging.AddFields(r.Context(), logging.Fields{logging.UserFieldKey: user.Username})
+				ctx := r.Context()
 				r = r.WithContext(auth.WithUser(ctx, user))
 			}
 			next.ServeHTTP(w, r)

diff --git a/pkg/api/serve.go b/pkg/api/serve.go
@@ -51,7 +51,8 @@ func Serve(cfg *config.Config, catalog *catalog.Catalog, middlewareAuthenticator
 			httputil.RequestIDHeaderName,
 			logging.Fields{logging.ServiceNameFieldKey: LoggerServiceName},
 			cfg.Logging.AuditLogLevel,
-			cfg.Logging.TraceRequestHeaders),
+			cfg.Logging.TraceRequestHeaders,
+			cfg.IsAdvancedAuth()),
 		AuthMiddleware(logger, swagger, middlewareAuthenticator, authService, sessionStore, &oidcConfig, &cookieAuthConfig),
 		MetricsMiddleware(swagger),
 	)

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -582,6 +582,10 @@ func (c *Config) IsExternalPrincipalsEnabled() bool {
 	return c.IsAuthTypeAPI() && c.Auth.AuthenticationAPI.ExternalPrincipalsEnabled
 }
 
+func (c *Config) IsAdvancedAuth() bool {
+	return c.IsAuthTypeAPI() && !c.IsAuthUISimplified()
+}
+
 func (c *Config) UISnippets() []apiparams.CodeSnippet {
 	snippets := make([]apiparams.CodeSnippet, 0, len(c.UI.Snippets))
 	for _, item := range c.UI.Snippets {

diff --git a/pkg/gateway/handler.go b/pkg/gateway/handler.go
@@ -62,7 +62,7 @@ type ServerContext struct {
 	verifyUnsupported bool
 }
 
-func NewHandler(region string, catalog *catalog.Catalog, multipartTracker multipart.Tracker, blockStore block.Adapter, authService auth.GatewayService, bareDomains []string, stats stats.Collector, pathProvider upload.PathProvider, fallbackURL *url.URL, auditLogLevel string, traceRequestHeaders bool, verifyUnsupported bool) http.Handler {
+func NewHandler(region string, catalog *catalog.Catalog, multipartTracker multipart.Tracker, blockStore block.Adapter, authService auth.GatewayService, bareDomains []string, stats stats.Collector, pathProvider upload.PathProvider, fallbackURL *url.URL, auditLogLevel string, traceRequestHeaders bool, verifyUnsupported bool, isAdvancedAuth bool) http.Handler {
 	var fallbackHandler http.Handler
 	if fallbackURL != nil {
 		fallbackProxy := gohttputil.NewSingleHostReverseProxy(fallbackURL)
@@ -112,20 +112,23 @@ func NewHandler(region string, catalog *catalog.Catalog, multipartTracker multip
 		"X-Amz-Request-Id",
 		logging.Fields{"service_name": "s3_gateway"},
 		auditLogLevel,
-		traceRequestHeaders)
+		traceRequestHeaders,
+		isAdvancedAuth)
 
 	h = loggingMiddleware(h)
 
-	h = EnrichWithOperation(sc,
-		DurationHandler(
-			AuthenticationHandler(authService, EnrichWithParts(bareDomains,
-				EnrichWithRepositoryOrFallback(catalog, authService, fallbackHandler,
-					OperationLookupHandler(
-						h))))))
-	logging.ContextUnavailable().WithFields(logging.Fields{
-		"s3_bare_domain": bareDomains,
-		"s3_region":      region,
-	}).Info("initialized S3 Gateway handler")
+	if isAdvancedAuth {
+		h = EnrichWithOperation(sc,
+			DurationHandler(
+				AuthenticationHandler(authService, EnrichWithParts(bareDomains,
+					EnrichWithRepositoryOrFallback(catalog, authService, fallbackHandler,
+						OperationLookupHandler(
+							h))))))
+		logging.ContextUnavailable().WithFields(logging.Fields{
+			"s3_bare_domain": bareDomains,
+			"s3_region":      region,
+		}).Info("initialized S3 Gateway handler")
+	}
 	return h
 }
 

diff --git a/pkg/gateway/testutil/gateway_setup.go b/pkg/gateway/testutil/gateway_setup.go
@@ -63,7 +63,7 @@ func GetBasicHandler(t *testing.T, authService *FakeAuthService, repoName string
 	_, err = c.CreateRepository(ctx, repoName, storageNamespace, "main", false)
 	testutil.Must(t, err)
 
-	handler := gateway.NewHandler(authService.Region, c, multipartTracker, blockAdapter, authService, []string{authService.BareDomain}, &stats.NullCollector{}, upload.DefaultPathProvider, nil, config.DefaultLoggingAuditLogLevel, true, false)
+	handler := gateway.NewHandler(authService.Region, c, multipartTracker, blockAdapter, authService, []string{authService.BareDomain}, &stats.NullCollector{}, upload.DefaultPathProvider, nil, config.DefaultLoggingAuditLogLevel, true, false, false)
 
 	return handler, &Dependencies{
 		blocks:  blockAdapter,

diff --git a/pkg/httputil/logging.go b/pkg/httputil/logging.go
@@ -63,7 +63,7 @@ func SourceIP(r *http.Request) string {
 	return sourceIP + ":" + sourcePort
 }
 
-func DefaultLoggingMiddleware(requestIDHeaderName string, fields logging.Fields, middlewareLogLevel string) func(next http.Handler) http.Handler {
+func DefaultLoggingMiddleware(requestIDHeaderName string, fields logging.Fields, middlewareLogLevel string, isAdvancedAuth bool) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			startTime := time.Now()
@@ -74,25 +74,31 @@ func DefaultLoggingMiddleware(requestIDHeaderName string, fields logging.Fields,
 
 			// add default fields to context
 			requestFields := logging.Fields{
-				logging.PathFieldKey:      r.RequestURI,
-				logging.MethodFieldKey:    r.Method,
-				logging.HostFieldKey:      r.Host,
-				logging.RequestIDFieldKey: reqID,
+				logging.PathFieldKey:   r.RequestURI,
+				logging.MethodFieldKey: r.Method,
+				logging.HostFieldKey:   r.Host,
 			}
-			for k, v := range fields {
-				requestFields[k] = v
+			if isAdvancedAuth {
+				requestFields[logging.RequestIDFieldKey] = reqID
+				for k, v := range fields {
+					requestFields[k] = v
+				}
 			}
 			r = r.WithContext(logging.AddFields(r.Context(), requestFields))
 			writer.Header().Set(requestIDHeaderName, reqID)
 			next.ServeHTTP(writer, r) // handle the request
 
 			loggingFields := logging.Fields{
-				"took":           time.Since(startTime),
-				"status_code":    writer.StatusCode,
-				"sent_bytes":     writer.ResponseSize,
-				"client":         client,
-				logging.LogAudit: true,
-				"source_ip":      sourceIP,
+				"start_time":  startTime,
+				"end_time":    time.Now(),
+				"took":        time.Since(startTime),
+				"status_code": writer.StatusCode,
+				"source_ip":   sourceIP,
+			}
+			if isAdvancedAuth {
+				loggingFields["sent_bytes"] = writer.ResponseSize
+				loggingFields["client"] = client
+				loggingFields[logging.LogAudit] = true
 			}
 
 			logLevel := strings.ToLower(middlewareLogLevel)
@@ -106,9 +112,9 @@ func DefaultLoggingMiddleware(requestIDHeaderName string, fields logging.Fields,
 	}
 }
 
-func LoggingMiddleware(requestIDHeaderName string, fields logging.Fields, loggingMiddlewareLevel string, traceRequestHeaders bool) func(next http.Handler) http.Handler {
+func LoggingMiddleware(requestIDHeaderName string, fields logging.Fields, loggingMiddlewareLevel string, traceRequestHeaders bool, isAdvancedAuth bool) func(next http.Handler) http.Handler {
 	if strings.ToLower(loggingMiddlewareLevel) == "trace" {
-		return TracingMiddleware(requestIDHeaderName, fields, traceRequestHeaders)
+		return TracingMiddleware(requestIDHeaderName, fields, traceRequestHeaders, isAdvancedAuth)
 	}
-	return DefaultLoggingMiddleware(requestIDHeaderName, fields, loggingMiddlewareLevel)
+	return DefaultLoggingMiddleware(requestIDHeaderName, fields, loggingMiddlewareLevel, isAdvancedAuth)
 }
diff --git a/pkg/httputil/tracing.go b/pkg/httputil/tracing.go
@@ -107,22 +107,26 @@ func presentBody(body []byte) string {
 	return string(body)
 }
 
-func TracingMiddleware(requestIDHeaderName string, fields logging.Fields, traceRequestHeaders bool) func(http.Handler) http.Handler {
+func TracingMiddleware(requestIDHeaderName string, fields logging.Fields, traceRequestHeaders bool, isAdvancedAuth bool) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			startTime := time.Now()
 			responseWriter := newResponseTracingWriter(w, RequestTracingMaxResponseBodySize)
 			r, reqID := RequestID(r)
+			client := GetRequestLakeFSClient(r)
+			sourceIP := SourceIP(r)
 
 			// add default fields to context
 			requestFields := logging.Fields{
-				logging.PathFieldKey:      r.RequestURI,
-				logging.MethodFieldKey:    r.Method,
-				logging.HostFieldKey:      r.Host,
-				logging.RequestIDFieldKey: reqID,
+				logging.PathFieldKey:   r.RequestURI,
+				logging.MethodFieldKey: r.Method,
+				logging.HostFieldKey:   r.Host,
 			}
-			for k, v := range fields {
-				requestFields[k] = v
+			if isAdvancedAuth {
+				requestFields[logging.RequestIDFieldKey] = reqID
+				for k, v := range fields {
+					requestFields[k] = v
+				}
 			}
 			r = r.WithContext(logging.AddFields(r.Context(), requestFields))
 			responseWriter.Header().Set(requestIDHeaderName, reqID)
@@ -134,15 +138,21 @@ func TracingMiddleware(requestIDHeaderName string, fields logging.Fields, traceR
 			next.ServeHTTP(responseWriter, r) // handle the request
 
 			traceFields := logging.Fields{
-				"took":             time.Since(startTime),
-				"status_code":      responseWriter.StatusCode,
-				"sent_bytes":       responseWriter.ResponseSize,
-				"request_body":     presentBody(requestBodyTracer.bodyRecorder.Buffer),
-				"response_body":    presentBody(responseWriter.BodyRecorder.Buffer),
-				"response_headers": responseWriter.Header(),
+				"start_time":  startTime,
+				"end_time":    time.Now(),
+				"took":        time.Since(startTime),
+				"status_code": responseWriter.StatusCode,
+				"source_ip":   sourceIP,
 			}
-			if traceRequestHeaders {
-				traceFields["request_headers"] = r.Header
+			if isAdvancedAuth {
+				traceFields["sent_bytes"] = responseWriter.ResponseSize
+				traceFields["client"] = client
+				traceFields["request_body"] = presentBody(requestBodyTracer.bodyRecorder.Buffer)
+				traceFields["response_body"] = presentBody(responseWriter.BodyRecorder.Buffer)
+				traceFields["response_headers"] = responseWriter.Header()
+				if traceRequestHeaders {
+					traceFields["request_headers"] = r.Header
+				}
 			}
 			logging.FromContext(r.Context()).
 				WithFields(traceFields).