Reduce user-k8s-client caching (#1778)

* Reduce user-k8s-client caching

- When a cluster comes up we can cache the client immediately with
  missing Flux CRDs, apps/sources won't work. System doesn't recover
  until the client expires in 30mins
- Also add extraEnvVars for users to use that doesn't wipe out default
  envVars

* Fixes missing end

* Adds some logging

* Fixes userNamespace cache cleanup

- Was only every 5mins

* main is up to date now

charts/mccp/templates/clusters-service/deployment.yaml

@@ -71,6 +71,11 @@ spec:
               {{- toYaml . | nindent 12 }}
             {{- end }}
             {{- end }}
+            {{- if .Values.extraEnvVars }}
+            {{- with .Values.extraEnvVars }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+            {{- end }}
           volumeMounts:
             - name: ui-server-volume
               mountPath: /html

charts/mccp/values.yaml

@@ -120,10 +120,10 @@ ingress:
   # kubernetes.io/ingress.class: nginx
   # kubernetes.io/tls-acme: "true"
   hosts:
-  - host: ""
-    paths:
-    - path: /
-      pathType: ImplementationSpecific
+    - host: ""
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
   #
   tls: []
   #  - secretName: chart-example-tls
@@ -135,6 +135,12 @@ envVars:
     value: "true"
   - name: WEAVE_GITOPS_FEATURE_CLUSTER
     value: "true"
+  - name: WEAVE_GITOPS_USERS_CLIENTS_TTL
+    value: "30s"
+
+# extraEnvVars is a list of extra environment variables to set on the
+# deployment. These are in addition to the envVars above.
+extraEnvVars: []
 
 networkPolicy:
   enabled: true

cmd/clusters-service/app/server.go

@@ -431,7 +431,10 @@ func StartServer(ctx context.Context, log logr.Logger, tempDir string, p Params)
 
 	clientsFactory := clustersmngr.CachedClientFactory
 	if !p.UseK8sCachedClients {
+		log.Info("Using un-cached clients")
 		clientsFactory = clustersmngr.ClientFactory
+	} else {
+		log.Info("Using cached clients")
 	}
 
 	clustersManager := clustersmngr.NewClustersManager(

pkg/cluster/namespaces/userCache.go

@@ -12,7 +12,7 @@ import (
 
 const (
 	cacheTTL            = time.Minute
-	cacheResolutionTime = 5 * time.Minute
+	cacheResolutionTime = 30 * time.Second
 )
 
 type UsersResourcesNamespaces struct {