Merge branch 'main' into add-jwt-authentication

.github/workflows/build-and-deploy.yaml

@@ -83,7 +83,7 @@ jobs:
       security-events: write
     steps:
       - name: Run Pharos
-        uses: kartverket/pharos@v0.3.0
+        uses: kartverket/pharos@v0.3.1
         with:
           image_url: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{needs.build.outputs.image_digest}}"
           tfsec: false

api/v1alpha1/application_types.go

@@ -3,14 +3,15 @@ package v1alpha1
 import (
 	"encoding/json"
 	"errors"
+	"time"
+
 	"github.com/kartverket/skiperator/api/v1alpha1/digdirator"
 	"github.com/kartverket/skiperator/api/v1alpha1/istiotypes"
 	"github.com/kartverket/skiperator/api/v1alpha1/podtypes"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"time"
 )
 
 // +kubebuilder:object:root=true
@@ -333,6 +334,16 @@ type PrometheusConfig struct {
 	//+kubebuilder:default:=false
 	//+kubebuilder:validation:Optional
 	AllowAllMetrics bool `json:"allowAllMetrics,omitempty"`
+
+	// ScrapeInterval specifies the interval at which Prometheus should scrape the metrics.
+	// The interval must be at least 15 seconds (if using "Xs") and divisible by 5.
+	// If minutes ("Xm") are used, the value must be at least 1m.
+	//
+	//+kubebuilder:default:="60s"
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:validation:XValidation:rule="self == '' || self.matches('^([0-9]+[sm])+$')",messageExpression="'Rejected: ' + self + ' as an invalid value. ScrapeInterval must be empty (default applies) or in the format of <number>s or <number>m.'"
+	//+kubebuilder:validation:XValidation:rule="self == '' || (self.endsWith('m') && int(self.split('m')[0]) >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >= 15 && int(self.split('s')[0]) % 5 == 0)",messageExpression="'Rejected: ' + self + ' as an invalid value. ScrapeInterval must be at least 15s (if using <s>) and divisible by 5, or at least 1m (if using <m>).'"
+	ScrapeInterval string `json:"scrapeInterval,omitempty"`
 }
 
 func NewDefaultReplicas() Replicas {

cmd/skiperator/main.go

@@ -3,16 +3,17 @@ package main
 import (
 	"flag"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/kartverket/skiperator/internal/controllers"
 	"github.com/kartverket/skiperator/internal/controllers/common"
 	"github.com/kartverket/skiperator/pkg/flags"
 	"github.com/kartverket/skiperator/pkg/k8sfeatures"
 	"github.com/kartverket/skiperator/pkg/resourceschemas"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
-	"os"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
-	"strings"
 
 	"go.uber.org/zap/zapcore"
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -54,6 +55,7 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
 		Development: !*isDeployment,
 		Level:       parsedLogLevel,
+		DestWriter:  os.Stdout,
 	})))
 
 	setupLog.Info(fmt.Sprintf("Running skiperator %s (commit %s)", Version, Commit))
@@ -69,13 +71,19 @@ func main() {
 
 	detectK8sVersion(kubeconfig)
 
+	pprofBindAddr := ""
+	if flags.FeatureFlags.EnableProfiling {
+		pprofBindAddr = ":8281"
+	}
+
 	mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{
 		Scheme:                  scheme,
 		HealthProbeBindAddress:  ":8081",
 		LeaderElection:          *leaderElection,
 		LeaderElectionNamespace: *leaderElectionNamespace,
 		Metrics:                 metricsserver.Options{BindAddress: ":8181"},
 		LeaderElectionID:        "skiperator",
+		PprofBindAddress:        pprofBindAddr,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

config/crd/skiperator.kartverket.no_applications.yaml

@@ -1081,6 +1081,24 @@ spec:
                     description: The port number or name where metrics are exposed
                       (at the Pod level).
                     x-kubernetes-int-or-string: true
+                  scrapeInterval:
+                    default: 60s
+                    description: |-
+                      ScrapeInterval specifies the interval at which Prometheus should scrape the metrics.
+                      The interval must be at least 15 seconds (if using "Xs") and divisible by 5.
+                      If minutes ("Xm") are used, the value must be at least 1m.
+                    type: string
+                    x-kubernetes-validations:
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be empty (default applies) or in
+                        the format of <number>s or <number>m.'''
+                      rule: self == '' || self.matches('^([0-9]+[sm])+$')
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be at least 15s (if using <s>)
+                        and divisible by 5, or at least 1m (if using <m>).'''
+                      rule: self == '' || (self.endsWith('m') && int(self.split('m')[0])
+                        >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >=
+                        15 && int(self.split('s')[0]) % 5 == 0)
                 required:
                 - port
                 type: object

config/crd/skiperator.kartverket.no_skipjobs.yaml

@@ -880,6 +880,24 @@ spec:
                     description: The port number or name where metrics are exposed
                       (at the Pod level).
                     x-kubernetes-int-or-string: true
+                  scrapeInterval:
+                    default: 60s
+                    description: |-
+                      ScrapeInterval specifies the interval at which Prometheus should scrape the metrics.
+                      The interval must be at least 15 seconds (if using "Xs") and divisible by 5.
+                      If minutes ("Xm") are used, the value must be at least 1m.
+                    type: string
+                    x-kubernetes-validations:
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be empty (default applies) or in
+                        the format of <number>s or <number>m.'''
+                      rule: self == '' || self.matches('^([0-9]+[sm])+$')
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be at least 15s (if using <s>)
+                        and divisible by 5, or at least 1m (if using <m>).'''
+                      rule: self == '' || (self.endsWith('m') && int(self.split('m')[0])
+                        >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >=
+                        15 && int(self.split('s')[0]) % 5 == 0)
                 required:
                 - port
                 type: object

go.mod

@@ -24,7 +24,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.32.1
 	k8s.io/apimachinery v0.32.1
 	k8s.io/client-go v0.32.1
-	sigs.k8s.io/controller-runtime v0.20.0
+	sigs.k8s.io/controller-runtime v0.20.1
 	sigs.k8s.io/controller-tools v0.17.1
 	sigs.k8s.io/kustomize/kyaml v0.19.0
 )

go.sum

@@ -1425,8 +1425,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
-sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrsfQkQs=
-sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU=
+sigs.k8s.io/controller-runtime v0.20.1 h1:JbGMAG/X94NeM3xvjenVUaBjy6Ui4Ogd/J5ZtjZnHaE=
+sigs.k8s.io/controller-runtime v0.20.1/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU=
 sigs.k8s.io/controller-tools v0.17.1 h1:bQ+dKCS7jY9AgpefenBDtm6geJZCHVKbegpLynxgyus=
 sigs.k8s.io/controller-tools v0.17.1/go.mod h1:3QXAdrmdxYuQ4MifvbCAFD9wLXn7jylnfBPYS4yVDdc=
 sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM=

internal/controllers/application.go

@@ -27,10 +27,10 @@ import (
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/maskinporten"
 	networkpolicy "github.com/kartverket/skiperator/pkg/resourcegenerator/networkpolicy/dynamic"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/pdb"
+	"github.com/kartverket/skiperator/pkg/resourcegenerator/prometheus"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/service"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/serviceaccount"
-	"github.com/kartverket/skiperator/pkg/resourcegenerator/servicemonitor"
 	"github.com/kartverket/skiperator/pkg/util"
 	nais_io_v1 "github.com/nais/liberator/pkg/apis/nais.io/v1"
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -215,7 +215,7 @@ func (r *ApplicationReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		defaultDenyAuthPolicy.Generate,
 		jwtAuthPolicy.Generate,
 		pdb.Generate,
-		servicemonitor.Generate,
+		prometheus.Generate,
 		idporten.Generate,
 		maskinporten.Generate,
 		requestauthentication.Generate,

internal/controllers/skipjob.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1"
 	"github.com/kartverket/skiperator/internal/controllers/common"
@@ -13,7 +14,7 @@ import (
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/istio/telemetry"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/job"
 	networkpolicy "github.com/kartverket/skiperator/pkg/resourcegenerator/networkpolicy/dynamic"
-	"github.com/kartverket/skiperator/pkg/resourcegenerator/podmonitor"
+	"github.com/kartverket/skiperator/pkg/resourcegenerator/prometheus"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/serviceaccount"
 	istionetworkingv1 "istio.io/client-go/pkg/apis/networking/v1"
@@ -36,6 +37,8 @@ const (
 	ConditionRunning  = "Running"
 	ConditionFinished = "Finished"
 	ConditionFailed   = "Failed"
+
+	skipJobFinalizer = "skip.statkart.no/finalizer"
 )
 
 // +kubebuilder:rbac:groups=skiperator.kartverket.no,resources=skipjobs;skipjobs/status,verbs=get;list;watch;update
@@ -97,6 +100,14 @@ func (r *SKIPJobReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return common.RequeueWithError(err)
 	}
 
+	isSkipJobMarkedToBeDeleted := skipJob.GetDeletionTimestamp() != nil
+	if isSkipJobMarkedToBeDeleted {
+		if err = r.finalizeSkipJob(skipJob, ctx); err != nil {
+			return ctrl.Result{}, err
+		}
+		return common.DoNotRequeue()
+	}
+
 	tmpSkipJob := skipJob.DeepCopy()
 	//TODO make sure we don't update the skipjob/application/routing after this step, it will cause endless reconciliations
 	//check that resource request limit 0.3 doesn't overwrite to 300m
@@ -121,9 +132,11 @@ func (r *SKIPJobReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{Requeue: true}, err
 	}
 
+	// Finalizer check is due to a bug when updating using controller-runtime
 	// If we update the SKIPJob initially on applied defaults before starting reconciling resources we allow all
 	// updates to be visible even though the controllerDuties may take some time.
-	if len(specDiff) > 0 {
+	if len(specDiff) > 0 || (!ctrlutil.ContainsFinalizer(tmpSkipJob, skipJobFinalizer) && ctrlutil.ContainsFinalizer(skipJob, skipJobFinalizer)) {
+		rLog.Debug("Queuing for spec diff")
 		err = r.GetClient().Update(ctx, skipJob)
 		return reconcile.Result{Requeue: true}, err
 	}
@@ -150,7 +163,7 @@ func (r *SKIPJobReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		serviceentry.Generate,
 		auth.Generate,
 		job.Generate,
-		podmonitor.Generate,
+		prometheus.Generate,
 		telemetry.Generate,
 	}
 
@@ -206,6 +219,10 @@ func (r *SKIPJobReconciler) setSKIPJobDefaults(ctx context.Context, skipJob *ski
 	if err := skipJob.FillDefaultSpec(); err != nil {
 		return fmt.Errorf("error when trying to fill default spec: %w", err)
 	}
+	if !ctrlutil.ContainsFinalizer(skipJob, skipJobFinalizer) {
+		ctrlutil.AddFinalizer(skipJob, skipJobFinalizer)
+	}
+
 	resourceutils.SetSKIPJobLabels(skipJob, skipJob)
 	skipJob.FillDefaultStatus()
 
@@ -222,6 +239,17 @@ func (r *SKIPJobReconciler) setResourceDefaults(resources []client.Object, skipJ
 	return nil
 }
 
+func (r *SKIPJobReconciler) finalizeSkipJob(skipJob *skiperatorv1alpha1.SKIPJob, ctx context.Context) error {
+	if ctrlutil.ContainsFinalizer(skipJob, skipJobFinalizer) {
+		ctrlutil.RemoveFinalizer(skipJob, skipJobFinalizer)
+		err := r.GetClient().Update(ctx, skipJob)
+		if err != nil {
+			return fmt.Errorf("something went wrong when trying to finalize SKIPJob: %w", err)
+		}
+	}
+	return nil
+}
+
 func (r *SKIPJobReconciler) getJobsToReconcile(ctx context.Context, object client.Object) []reconcile.Request {
 	var jobsToReconcile skiperatorv1alpha1.SKIPJobList
 	var reconcileRequests []reconcile.Request

pkg/flags/flags.go

@@ -14,11 +14,13 @@ var FeatureFlags *Features
 // environment variables.
 type Features struct {
 	DisablePodTopologySpreadConstraints bool
+	EnableProfiling                     bool
 }
 
 func init() {
 	FeatureFlags = &Features{
 		DisablePodTopologySpreadConstraints: getEnvWithFallback("DISABLE_PTSC", false),
+		EnableProfiling:                     getEnvWithFallback("ENABLE_PROFILING", false),
 	}
 }
 

pkg/resourcegenerator/podmonitor/pod_monitor.go → pkg/resourcegenerator/prometheus/pod_monitor.go

@@ -1,16 +1,21 @@
-package podmonitor
+package prometheus
 
 import (
 	"fmt"
+	"strings"
+
 	skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1"
 	"github.com/kartverket/skiperator/pkg/reconciliation"
 	"github.com/kartverket/skiperator/pkg/util"
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"strings"
 )
 
-func Generate(r reconciliation.Reconciliation) error {
+func init() {
+	multiGenerator.Register(reconciliation.JobType, generateForSkipJob)
+}
+
+func generateForSkipJob(r reconciliation.Reconciliation) error {
 	ctxLog := r.GetLogger()
 	ctxLog.Debug("Attempting to generate podmonitor for skipjob", "skipjob", r.GetSKIPObject().GetName())
 
@@ -41,6 +46,7 @@ func Generate(r reconciliation.Reconciliation) error {
 			{
 				Path:       util.IstioMetricsPath,
 				TargetPort: &util.IstioMetricsPortName,
+				Interval:   getScrapeInterval(skipJob.Spec.Prometheus),
 			},
 		},
 	}

pkg/resourcegenerator/prometheus/prometheus.go

@@ -0,0 +1,25 @@
+package prometheus
+
+import (
+	"github.com/kartverket/skiperator/api/v1alpha1"
+	"github.com/kartverket/skiperator/pkg/reconciliation"
+	"github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils/generator"
+	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+)
+
+var (
+	multiGenerator        = generator.NewMulti()
+	defaultScrapeInterval = pov1.Duration("60s")
+)
+
+func Generate(r reconciliation.Reconciliation) error {
+	return multiGenerator.Generate(r, "PrometheusCRD")
+}
+
+func getScrapeInterval(pc *v1alpha1.PrometheusConfig) pov1.Duration {
+	if pc == nil {
+		return defaultScrapeInterval
+	}
+
+	return pov1.Duration(pc.ScrapeInterval)
+}

pkg/resourcegenerator/servicemonitor/service_monitor.go → pkg/resourcegenerator/prometheus/service_monitor.go

@@ -1,16 +1,21 @@
-package servicemonitor
+package prometheus
 
 import (
 	"fmt"
+	"strings"
+
 	skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1"
 	"github.com/kartverket/skiperator/pkg/reconciliation"
 	"github.com/kartverket/skiperator/pkg/util"
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"strings"
 )
 
-func Generate(r reconciliation.Reconciliation) error {
+func init() {
+	multiGenerator.Register(reconciliation.ApplicationType, generateForApplication)
+}
+
+func generateForApplication(r reconciliation.Reconciliation) error {
 	ctxLog := r.GetLogger()
 	if r.GetType() != reconciliation.ApplicationType {
 		return fmt.Errorf("unsupported type %s in service monitor", r.GetType())
@@ -44,6 +49,7 @@ func Generate(r reconciliation.Reconciliation) error {
 			{
 				Path:       util.IstioMetricsPath,
 				TargetPort: &util.IstioMetricsPortName,
+				Interval:   getScrapeInterval(application.Spec.Prometheus),
 				MetricRelabelConfigs: []pov1.RelabelConfig{
 					{
 						Action:       "drop",

tests/application/service-monitor/application-istio-assert.yaml

@@ -68,6 +68,7 @@ spec:
   endpoints:
     - targetPort: istio-metrics
       path: /stats/prometheus
+      interval: "60s"
       metricRelabelings:
       - action: drop
         regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket