diff --git a/api/v1alpha1/application_types.go b/api/v1alpha1/application_types.go index 16a447d6..b6d3471e 100644 --- a/api/v1alpha1/application_types.go +++ b/api/v1alpha1/application_types.go @@ -3,6 +3,8 @@ package v1alpha1 import ( "encoding/json" "errors" + "time" + "github.com/kartverket/skiperator/api/v1alpha1/digdirator" "github.com/kartverket/skiperator/api/v1alpha1/istiotypes" "github.com/kartverket/skiperator/api/v1alpha1/podtypes" @@ -10,7 +12,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "time" ) // +kubebuilder:object:root=true @@ -333,6 +334,16 @@ type PrometheusConfig struct { //+kubebuilder:default:=false //+kubebuilder:validation:Optional AllowAllMetrics bool `json:"allowAllMetrics,omitempty"` + + // ScrapeInterval specifies the interval at which Prometheus should scrape the metrics. + // The interval must be at least 15 seconds (if using "Xs") and divisible by 5. + // If minutes ("Xm") are used, the value must be at least 1m. + // + //+kubebuilder:default:="60s" + //+kubebuilder:validation:Optional + //+kubebuilder:validation:XValidation:rule="self == '' || self.matches('^([0-9]+[sm])+$')",messageExpression="'Rejected: ' + self + ' as an invalid value. ScrapeInterval must be empty (default applies) or in the format of s or m.'" + //+kubebuilder:validation:XValidation:rule="self == '' || (self.endsWith('m') && int(self.split('m')[0]) >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >= 15 && int(self.split('s')[0]) % 5 == 0)",messageExpression="'Rejected: ' + self + ' as an invalid value. ScrapeInterval must be at least 15s (if using ) and divisible by 5, or at least 1m (if using ).'" + ScrapeInterval string `json:"scrapeInterval,omitempty"` } func NewDefaultReplicas() Replicas { diff --git a/cmd/skiperator/main.go b/cmd/skiperator/main.go index 65465786..e294da70 100644 --- a/cmd/skiperator/main.go +++ b/cmd/skiperator/main.go @@ -55,6 +55,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{ Development: !*isDeployment, Level: parsedLogLevel, + DestWriter: os.Stdout, }))) setupLog.Info(fmt.Sprintf("Running skiperator %s (commit %s)", Version, Commit)) diff --git a/config/crd/skiperator.kartverket.no_applications.yaml b/config/crd/skiperator.kartverket.no_applications.yaml index 344573d2..848690ec 100644 --- a/config/crd/skiperator.kartverket.no_applications.yaml +++ b/config/crd/skiperator.kartverket.no_applications.yaml @@ -947,6 +947,24 @@ spec: description: The port number or name where metrics are exposed (at the Pod level). x-kubernetes-int-or-string: true + scrapeInterval: + default: 60s + description: |- + ScrapeInterval specifies the interval at which Prometheus should scrape the metrics. + The interval must be at least 15 seconds (if using "Xs") and divisible by 5. + If minutes ("Xm") are used, the value must be at least 1m. + type: string + x-kubernetes-validations: + - messageExpression: '''Rejected: '' + self + '' as an invalid + value. ScrapeInterval must be empty (default applies) or in + the format of s or m.''' + rule: self == '' || self.matches('^([0-9]+[sm])+$') + - messageExpression: '''Rejected: '' + self + '' as an invalid + value. ScrapeInterval must be at least 15s (if using ) + and divisible by 5, or at least 1m (if using ).''' + rule: self == '' || (self.endsWith('m') && int(self.split('m')[0]) + >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >= + 15 && int(self.split('s')[0]) % 5 == 0) required: - port type: object diff --git a/config/crd/skiperator.kartverket.no_skipjobs.yaml b/config/crd/skiperator.kartverket.no_skipjobs.yaml index d56e6bc0..214263cd 100644 --- a/config/crd/skiperator.kartverket.no_skipjobs.yaml +++ b/config/crd/skiperator.kartverket.no_skipjobs.yaml @@ -880,6 +880,24 @@ spec: description: The port number or name where metrics are exposed (at the Pod level). x-kubernetes-int-or-string: true + scrapeInterval: + default: 60s + description: |- + ScrapeInterval specifies the interval at which Prometheus should scrape the metrics. + The interval must be at least 15 seconds (if using "Xs") and divisible by 5. + If minutes ("Xm") are used, the value must be at least 1m. + type: string + x-kubernetes-validations: + - messageExpression: '''Rejected: '' + self + '' as an invalid + value. ScrapeInterval must be empty (default applies) or in + the format of s or m.''' + rule: self == '' || self.matches('^([0-9]+[sm])+$') + - messageExpression: '''Rejected: '' + self + '' as an invalid + value. ScrapeInterval must be at least 15s (if using ) + and divisible by 5, or at least 1m (if using ).''' + rule: self == '' || (self.endsWith('m') && int(self.split('m')[0]) + >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >= + 15 && int(self.split('s')[0]) % 5 == 0) required: - port type: object diff --git a/internal/controllers/application.go b/internal/controllers/application.go index 9adcff0c..71f445cf 100644 --- a/internal/controllers/application.go +++ b/internal/controllers/application.go @@ -24,10 +24,10 @@ import ( "github.com/kartverket/skiperator/pkg/resourcegenerator/maskinporten" networkpolicy "github.com/kartverket/skiperator/pkg/resourcegenerator/networkpolicy/dynamic" "github.com/kartverket/skiperator/pkg/resourcegenerator/pdb" + "github.com/kartverket/skiperator/pkg/resourcegenerator/prometheus" "github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils" "github.com/kartverket/skiperator/pkg/resourcegenerator/service" "github.com/kartverket/skiperator/pkg/resourcegenerator/serviceaccount" - "github.com/kartverket/skiperator/pkg/resourcegenerator/servicemonitor" "github.com/kartverket/skiperator/pkg/util" nais_io_v1 "github.com/nais/liberator/pkg/apis/nais.io/v1" pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -207,7 +207,7 @@ func (r *ApplicationReconciler) Reconcile(ctx context.Context, req reconcile.Req networkpolicy.Generate, authorizationpolicy.Generate, pdb.Generate, - servicemonitor.Generate, + prometheus.Generate, idporten.Generate, maskinporten.Generate, deployment.Generate, diff --git a/internal/controllers/skipjob.go b/internal/controllers/skipjob.go index 5ccc5612..8822d259 100644 --- a/internal/controllers/skipjob.go +++ b/internal/controllers/skipjob.go @@ -14,7 +14,7 @@ import ( "github.com/kartverket/skiperator/pkg/resourcegenerator/istio/telemetry" "github.com/kartverket/skiperator/pkg/resourcegenerator/job" networkpolicy "github.com/kartverket/skiperator/pkg/resourcegenerator/networkpolicy/dynamic" - "github.com/kartverket/skiperator/pkg/resourcegenerator/podmonitor" + "github.com/kartverket/skiperator/pkg/resourcegenerator/prometheus" "github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils" "github.com/kartverket/skiperator/pkg/resourcegenerator/serviceaccount" istionetworkingv1 "istio.io/client-go/pkg/apis/networking/v1" @@ -163,7 +163,7 @@ func (r *SKIPJobReconciler) Reconcile(ctx context.Context, req reconcile.Request serviceentry.Generate, auth.Generate, job.Generate, - podmonitor.Generate, + prometheus.Generate, telemetry.Generate, } diff --git a/pkg/resourcegenerator/podmonitor/pod_monitor.go b/pkg/resourcegenerator/prometheus/pod_monitor.go similarity index 87% rename from pkg/resourcegenerator/podmonitor/pod_monitor.go rename to pkg/resourcegenerator/prometheus/pod_monitor.go index 816ea602..c871a89a 100644 --- a/pkg/resourcegenerator/podmonitor/pod_monitor.go +++ b/pkg/resourcegenerator/prometheus/pod_monitor.go @@ -1,16 +1,21 @@ -package podmonitor +package prometheus import ( "fmt" + "strings" + skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1" "github.com/kartverket/skiperator/pkg/reconciliation" "github.com/kartverket/skiperator/pkg/util" pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" ) -func Generate(r reconciliation.Reconciliation) error { +func init() { + multiGenerator.Register(reconciliation.JobType, generateForSkipJob) +} + +func generateForSkipJob(r reconciliation.Reconciliation) error { ctxLog := r.GetLogger() ctxLog.Debug("Attempting to generate podmonitor for skipjob", "skipjob", r.GetSKIPObject().GetName()) @@ -41,6 +46,7 @@ func Generate(r reconciliation.Reconciliation) error { { Path: util.IstioMetricsPath, TargetPort: &util.IstioMetricsPortName, + Interval: getScrapeInterval(skipJob.Spec.Prometheus), }, }, } diff --git a/pkg/resourcegenerator/prometheus/prometheus.go b/pkg/resourcegenerator/prometheus/prometheus.go new file mode 100644 index 00000000..76a7a0b4 --- /dev/null +++ b/pkg/resourcegenerator/prometheus/prometheus.go @@ -0,0 +1,25 @@ +package prometheus + +import ( + "github.com/kartverket/skiperator/api/v1alpha1" + "github.com/kartverket/skiperator/pkg/reconciliation" + "github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils/generator" + pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +var ( + multiGenerator = generator.NewMulti() + defaultScrapeInterval = pov1.Duration("60s") +) + +func Generate(r reconciliation.Reconciliation) error { + return multiGenerator.Generate(r, "PrometheusCRD") +} + +func getScrapeInterval(pc *v1alpha1.PrometheusConfig) pov1.Duration { + if pc == nil { + return defaultScrapeInterval + } + + return pov1.Duration(pc.ScrapeInterval) +} diff --git a/pkg/resourcegenerator/servicemonitor/service_monitor.go b/pkg/resourcegenerator/prometheus/service_monitor.go similarity index 88% rename from pkg/resourcegenerator/servicemonitor/service_monitor.go rename to pkg/resourcegenerator/prometheus/service_monitor.go index 205ebb9e..e1d71041 100644 --- a/pkg/resourcegenerator/servicemonitor/service_monitor.go +++ b/pkg/resourcegenerator/prometheus/service_monitor.go @@ -1,16 +1,21 @@ -package servicemonitor +package prometheus import ( "fmt" + "strings" + skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1" "github.com/kartverket/skiperator/pkg/reconciliation" "github.com/kartverket/skiperator/pkg/util" pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" ) -func Generate(r reconciliation.Reconciliation) error { +func init() { + multiGenerator.Register(reconciliation.ApplicationType, generateForApplication) +} + +func generateForApplication(r reconciliation.Reconciliation) error { ctxLog := r.GetLogger() if r.GetType() != reconciliation.ApplicationType { return fmt.Errorf("unsupported type %s in service monitor", r.GetType()) @@ -44,6 +49,7 @@ func Generate(r reconciliation.Reconciliation) error { { Path: util.IstioMetricsPath, TargetPort: &util.IstioMetricsPortName, + Interval: getScrapeInterval(application.Spec.Prometheus), MetricRelabelConfigs: []pov1.RelabelConfig{ { Action: "drop", diff --git a/tests/application/service-monitor/application-istio-assert.yaml b/tests/application/service-monitor/application-istio-assert.yaml index 63fbfc36..610183c4 100644 --- a/tests/application/service-monitor/application-istio-assert.yaml +++ b/tests/application/service-monitor/application-istio-assert.yaml @@ -68,6 +68,7 @@ spec: endpoints: - targetPort: istio-metrics path: /stats/prometheus + interval: "60s" metricRelabelings: - action: drop regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket diff --git a/tests/application/service-monitor/application-simple-assert.yaml b/tests/application/service-monitor/application-simple-assert.yaml index 565581e1..35511fe1 100644 --- a/tests/application/service-monitor/application-simple-assert.yaml +++ b/tests/application/service-monitor/application-simple-assert.yaml @@ -57,6 +57,7 @@ spec: endpoints: - targetPort: istio-metrics path: /stats/prometheus + interval: "60s" metricRelabelings: - action: drop regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket @@ -100,4 +101,4 @@ spec: app.kubernetes.io/instance: alloy app.kubernetes.io/name: alloy ports: - - port: istio-metrics \ No newline at end of file + - port: istio-metrics diff --git a/tests/application/service-monitor/application-simple-custom-interval-assert.yaml b/tests/application/service-monitor/application-simple-custom-interval-assert.yaml new file mode 100644 index 00000000..1b3420d1 --- /dev/null +++ b/tests/application/service-monitor/application-simple-custom-interval-assert.yaml @@ -0,0 +1,23 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + instance: primary + name: some-simple-monitored-app-1 + namespace: sm-istio-ns-2 +spec: + endpoints: + - targetPort: istio-metrics + path: /stats/prometheus + interval: "90s" + metricRelabelings: + - action: drop + regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket + sourceLabels: + - __name__ + selector: + matchLabels: + app: some-simple-monitored-app-1 + namespaceSelector: + matchNames: + - sm-istio-ns-2 diff --git a/tests/application/service-monitor/application-simple-custom-interval-invalid.yaml b/tests/application/service-monitor/application-simple-custom-interval-invalid.yaml new file mode 100644 index 00000000..1fa5fb5f --- /dev/null +++ b/tests/application/service-monitor/application-simple-custom-interval-invalid.yaml @@ -0,0 +1,11 @@ +apiVersion: skiperator.kartverket.no/v1alpha1 +kind: Application +metadata: + name: some-simple-monitored-app-1 + namespace: sm-istio-ns-2 +spec: + image: image + port: 8080 + prometheus: + port: 8080 + scrapeInterval: 10s diff --git a/tests/application/service-monitor/application-simple-custom-interval.yaml b/tests/application/service-monitor/application-simple-custom-interval.yaml new file mode 100644 index 00000000..d585628e --- /dev/null +++ b/tests/application/service-monitor/application-simple-custom-interval.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sm-istio-ns-2 + labels: + istio.io/rev: "revision-1" +--- +apiVersion: skiperator.kartverket.no/v1alpha1 +kind: Application +metadata: + name: some-simple-monitored-app-1 + namespace: sm-istio-ns-2 +spec: + image: image + port: 8080 + prometheus: + port: 8080 + scrapeInterval: 90s diff --git a/tests/application/service-monitor/chainsaw-test.yaml b/tests/application/service-monitor/chainsaw-test.yaml index 4a33eec9..60e7d834 100644 --- a/tests/application/service-monitor/chainsaw-test.yaml +++ b/tests/application/service-monitor/chainsaw-test.yaml @@ -23,3 +23,10 @@ spec: file: patch-application-allowall.yaml - assert: file: patch-application-allowall-assert.yaml + - try: + - apply: + file: application-simple-custom-interval.yaml + - assert: + file: application-simple-custom-interval-assert.yaml + - error: + file: application-simple-custom-interval-invalid.yaml diff --git a/tests/application/service-monitor/patch-application-allowall-assert.yaml b/tests/application/service-monitor/patch-application-allowall-assert.yaml index 90844392..a5eff8db 100644 --- a/tests/application/service-monitor/patch-application-allowall-assert.yaml +++ b/tests/application/service-monitor/patch-application-allowall-assert.yaml @@ -9,6 +9,7 @@ spec: endpoints: - targetPort: istio-metrics path: /stats/prometheus + interval: "60s" selector: matchLabels: app: some-monitored-app-1 diff --git a/tests/skipjob/podmonitor/chainsaw-test.yaml b/tests/skipjob/podmonitor/chainsaw-test.yaml index 2c206ad0..23008978 100644 --- a/tests/skipjob/podmonitor/chainsaw-test.yaml +++ b/tests/skipjob/podmonitor/chainsaw-test.yaml @@ -18,6 +18,11 @@ spec: file: patch-skipjob-with-monitoring.yaml - assert: file: patch-skipjob-with-monitoring-assert.yaml + - try: + - patch: + file: patch-skipjob-with-monitoring-custom-interval.yaml + - assert: + file: patch-skipjob-with-monitoring-custom-interval-assert.yaml - try: - patch: file: patch-skipjob-remove-monitoring.yaml diff --git a/tests/skipjob/podmonitor/patch-skipjob-remove-monitoring-error.yaml b/tests/skipjob/podmonitor/patch-skipjob-remove-monitoring-error.yaml index dbcdd72f..50c9be59 100644 --- a/tests/skipjob/podmonitor/patch-skipjob-remove-monitoring-error.yaml +++ b/tests/skipjob/podmonitor/patch-skipjob-remove-monitoring-error.yaml @@ -11,6 +11,7 @@ spec: podMetricsEndpoints: - targetPort: istio-metrics path: "/stats/prometheus" + interval: "60s" metricRelabelings: - action: drop regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket diff --git a/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-assert.yaml b/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-assert.yaml index 0bea5917..90d54f96 100644 --- a/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-assert.yaml +++ b/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-assert.yaml @@ -11,6 +11,7 @@ spec: podMetricsEndpoints: - targetPort: istio-metrics path: "/stats/prometheus" + interval: "60s" selector: matchLabels: app: podmonitor diff --git a/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval-assert.yaml b/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval-assert.yaml new file mode 100644 index 00000000..227c9a96 --- /dev/null +++ b/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + instance: primary + name: podmonitor-monitor +spec: + namespaceSelector: + matchNames: + - podmonitor-ns + podMetricsEndpoints: + - targetPort: istio-metrics + path: "/stats/prometheus" + interval: "100s" + selector: + matchLabels: + app: podmonitor diff --git a/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval.yaml b/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval.yaml new file mode 100644 index 00000000..16d1e2c1 --- /dev/null +++ b/tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval.yaml @@ -0,0 +1,17 @@ +apiVersion: skiperator.kartverket.no/v1alpha1 +kind: SKIPJob +metadata: + name: podmonitor +spec: + container: + image: "perl:5.34.0" + command: + - "perl" + - "-Mbignum=bpi" + - "-wle" + - "print bpi(2000)" + prometheus: + path: /metrics + port: 8080 + allowAllMetrics: true + scrapeInterval: "100s" diff --git a/tests/skipjob/podmonitor/skipjob-with-monitoring-assert.yaml b/tests/skipjob/podmonitor/skipjob-with-monitoring-assert.yaml index 77549aac..a0afca8c 100644 --- a/tests/skipjob/podmonitor/skipjob-with-monitoring-assert.yaml +++ b/tests/skipjob/podmonitor/skipjob-with-monitoring-assert.yaml @@ -55,6 +55,7 @@ spec: podMetricsEndpoints: - targetPort: istio-metrics path: "/stats/prometheus" + interval: "60s" metricRelabelings: - action: drop regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket