changed logging to only support syslogng and elasticsearch.

When a user wants logging, now automatcially SyslogNGOutput and SyslogNGFlow objects are created. In addition, support for Splunk has been removed and only Elasticsearch Outputs are supported. A test case has been modified due to a known issue whiich should be fixed in the next release of logging-operator (#1140) Signed-off-by: mzeevi <meytar80@gmail.com>
dana-team · Apr 24, 2024 · 589dcf8 · 589dcf8
1 parent 29a212a
commit 589dcf8
Show file tree

Hide file tree

Showing 35 changed files with 609 additions and 591 deletions.
diff --git a/Makefile b/Makefile
@@ -167,7 +167,8 @@ enable-nfs-knative: ## Enable NFS for Knative
 
 .PHONY: install-logging
 install-logging: ## Install logging operator on the kind cluster
-	helm upgrade --install --wait --create-namespace --namespace logging logging-operator oci://ghcr.io/kube-logging/helm-charts/logging-operator --set logging.enabled=true
+	helm upgrade --install --wait --create-namespace --namespace logging-operator-system logging-operator oci://ghcr.io/kube-logging/helm-charts/logging-operator
+	kubectl apply -f hack/logging-operator-resources.yaml
 
 KNATIVE_URL ?= https://github.com/knative-extensions/kn-plugin-quickstart/releases/download/knative-v1.11.2/kn-quickstart-linux-amd64
 KNATIVE_HPA_URL ?= https://github.com/knative/serving/releases/download/knative-v1.11.2/serving-hpa.yaml

diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ The `container-app-operator` project can work as a standalone solution, but is m
 - [x] Support for autoscaler (`HPA` or `KPA`) according to the chosen `scaleMetric` (`concurrency`, `rps`, `cpu`, `memory`) with default settings.
 - [x] Support for HTTP/HTTPS `DomainMapping` for accessing applications via `Ingress`/`Route`.
 - [x] Support for all `Knative Serving` configurations.
-- [x] Support for exporting logs to `Elasticsearch` and `Splunk` indexes.
+- [x] Support for exporting logs to an `Elasticsearch` index.
 - [x] Support for changing the state of `Capp` from `enabled` (workload is in running state) to `disabled` (workload is not in running state).
 - [x] Support for external NFS storage connected to `Capp` by using `volumeMounts`.
 
@@ -136,9 +136,8 @@ spec:
     type: elastic
     host: 10.11.12.13
     index: main
-    username: elastic
-    passwordSecretName: es-elastic-user
-    sslVerify: false
+    user: elastic
+    passwordSecret: es-elastic-user
   scaleMetric: concurrency
   state: enabled
 ```
diff --git a/api/v1alpha1/capp_types.go b/api/v1alpha1/capp_types.go
@@ -107,35 +107,26 @@ type RouteSpec struct {
 // LogSpec defines the configuration for shipping Capp logs.
 type LogSpec struct {
 	// Type defines where to send the Capp logs
-	// Possible values : "elastic" and "splunk".
+	// +kubebuilder:validation:Enum=elastic
 	// +optional
 	Type string `json:"type,omitempty"`
 
 	// Host defines Elasticsearch or Splunk host.
 	// +optional
 	Host string `json:"host,omitempty"`
 
-	// SSLVerify determines whether to skip ssl verification.
-	// +optional
-	SSLVerify bool `json:"sslVerify,omitempty"`
-
 	// Index defines the index name to write events to.
 	// +optional
 	Index string `json:"index,omitempty"`
 
-	// UserName defines a User for authentication.
+	// User defines a User for authentication.
 	// +optional
-	UserName string `json:"username,omitempty"`
+	User string `json:"username,omitempty"`
 
-	// PasswordSecretName defines the name of the secret
+	// PasswordSecret defines the name of the secret
 	// containing the password for authentication.
 	// +optional
-	PasswordSecretName string `json:"passwordSecretName,omitempty"`
-
-	// HecTokenSecretName defines the name of the secret
-	// containing the Splunk Hec token.
-	// +optional
-	HecTokenSecretName string `json:"hecTokenSecretName,omitempty"`
+	PasswordSecret string `json:"passwordSecretName,omitempty"`
 }
 
 // ApplicationLinks contains relevant information about
@@ -171,17 +162,17 @@ type StateStatus struct {
 	LastChange metav1.Time `json:"lastChange,omitempty"`
 }
 
-// LoggingStatus defines the state of the Flow and Output objects linked to the Capp.
+// LoggingStatus defines the state of the SyslogNGFlow and SyslogNGOutput objects linked to the Capp.
 type LoggingStatus struct {
-	// Flow represents the Status of the Flow used by the Capp.
+	// SyslogNGFlow represents the Status of the SyslogNGFlow used by the Capp.
 	// +optional
-	Flow loggingv1beta1.FlowStatus `json:"flow,omitempty"`
+	SyslogNGFlow loggingv1beta1.SyslogNGFlowStatus `json:"syslogngflow,omitempty"`
 
-	// Output represents the Status of the Output used by the Capp.
+	// SyslogNGOutput represents the Status of the SyslogNGOutput used by the Capp.
 	// +optional
-	Output loggingv1beta1.OutputStatus `json:"output,omitempty"`
+	SyslogNGOutput loggingv1beta1.SyslogNGOutputStatus `json:"syslogngoutput,omitempty"`
 
-	// Conditions contain details about the current state of the Output and Flow used by the Capp.
+	// Conditions contain details about the current state of the SyslogNGFlow and SyslogNGOutput used by the Capp.
 	// +optional
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 }

diff --git a/api/v1alpha1/capprevision_types.go b/api/v1alpha1/capprevision_types.go
@@ -42,7 +42,7 @@ type CappTemplate struct {
 	// +optional
 	Labels map[string]string `json:"labels,omitempty"`
 
-	// Annotations is the related Capp annotations
+	// Annotations is a map of string keys and values which are the actual annotations of the related Capp
 	// +optional
 	Annotations map[string]string `json:"Annotations,omitempty"`
 }

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/cmd/main.go b/cmd/main.go
@@ -20,9 +20,9 @@ import (
 	"flag"
 	"os"
 
-	crcontroller "github.com/dana-team/container-app-operator/internal/kinds/capp-revision/controllers"
 	cappcontroller "github.com/dana-team/container-app-operator/internal/kinds/capp/controllers"
 	"github.com/dana-team/container-app-operator/internal/kinds/capp/utils"
+	crcontroller "github.com/dana-team/container-app-operator/internal/kinds/capprevision/controllers"
 
 	nfspvcv1alpha1 "github.com/dana-team/nfspvc-operator/api/v1alpha1"
 	"github.com/go-logr/zapr"

diff --git a/config/crd/bases/rcs.dana.io_capprevisions.yaml b/config/crd/bases/rcs.dana.io_capprevisions.yaml
@@ -46,7 +46,8 @@ spec:
                   Annotations:
                     additionalProperties:
                       type: string
-                    description: Annotations is the related Capp annotations
+                    description: Annotations is a map of string keys and values which
+                      are the actual annotations of the related Capp
                     type: object
                   cappSpec:
                     description: Spec is the related Capp spec
@@ -7832,11 +7833,6 @@ spec:
                         description: LogSpec defines the configuration for shipping
                           Capp logs.
                         properties:
-                          hecTokenSecretName:
-                            description: |-
-                              HecTokenSecretName defines the name of the secret
-                              containing the Splunk Hec token.
-                            type: string
                           host:
                             description: Host defines Elasticsearch or Splunk host.
                             type: string
@@ -7846,20 +7842,16 @@ spec:
                             type: string
                           passwordSecretName:
                             description: |-
-                              PasswordSecretName defines the name of the secret
+                              PasswordSecret defines the name of the secret
                               containing the password for authentication.
                             type: string
-                          sslVerify:
-                            description: SSLVerify determines whether to skip ssl
-                              verification.
-                            type: boolean
                           type:
-                            description: |-
-                              Type defines where to send the Capp logs
-                              Possible values : "elastic" and "splunk".
+                            description: Type defines where to send the Capp logs
+                            enum:
+                            - elastic
                             type: string
                           username:
-                            description: UserName defines a User for authentication.
+                            description: User defines a User for authentication.
                             type: string
                         type: object
                       routeSpec:

diff --git a/config/crd/bases/rcs.dana.io_capps.yaml b/config/crd/bases/rcs.dana.io_capps.yaml
@@ -7642,11 +7642,6 @@ spec:
               logSpec:
                 description: LogSpec defines the configuration for shipping Capp logs.
                 properties:
-                  hecTokenSecretName:
-                    description: |-
-                      HecTokenSecretName defines the name of the secret
-                      containing the Splunk Hec token.
-                    type: string
                   host:
                     description: Host defines Elasticsearch or Splunk host.
                     type: string
@@ -7655,19 +7650,16 @@ spec:
                     type: string
                   passwordSecretName:
                     description: |-
-                      PasswordSecretName defines the name of the secret
+                      PasswordSecret defines the name of the secret
                       containing the password for authentication.
                     type: string
-                  sslVerify:
-                    description: SSLVerify determines whether to skip ssl verification.
-                    type: boolean
                   type:
-                    description: |-
-                      Type defines where to send the Capp logs
-                      Possible values : "elastic" and "splunk".
+                    description: Type defines where to send the Capp logs
+                    enum:
+                    - elastic
                     type: string
                   username:
-                    description: UserName defines a User for authentication.
+                    description: User defines a User for authentication.
                     type: string
                 type: object
               routeSpec:
@@ -8165,7 +8157,7 @@ spec:
                 properties:
                   conditions:
                     description: Conditions contain details about the current state
-                      of the Output and Flow used by the Capp.
+                      of the SyslogNGFlow and SyslogNGOutput used by the Capp.
                     items:
                       description: "Condition contains details for one aspect of the
                         current state of this API Resource.\n---\nThis struct is intended
@@ -8236,9 +8228,9 @@ spec:
                       - type
                       type: object
                     type: array
-                  flow:
-                    description: Flow represents the Status of the Flow used by the
-                      Capp.
+                  syslogngflow:
+                    description: SyslogNGFlow represents the Status of the SyslogNGFlow
+                      used by the Capp.
                     properties:
                       active:
                         type: boolean
@@ -8249,9 +8241,9 @@ spec:
                       problemsCount:
                         type: integer
                     type: object
-                  output:
-                    description: Output represents the Status of the Output used by
-                      the Capp.
+                  syslogngoutput:
+                    description: SyslogNGOutput represents the Status of the SyslogNGOutput
+                      used by the Capp.
                     properties:
                       active:
                         type: boolean

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -1,2 +1,8 @@
 resources:
 - manager.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+  newName: controller
+  newTag: latest
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -53,7 +53,7 @@ rules:
 - apiGroups:
   - logging.banzaicloud.io
   resources:
-  - flows
+  - syslogngflows
   verbs:
   - create
   - delete
@@ -64,7 +64,7 @@ rules:
 - apiGroups:
   - logging.banzaicloud.io
   resources:
-  - outputs
+  - syslogngoutputs
   verbs:
   - create
   - delete

diff --git a/go.mod b/go.mod
@@ -7,11 +7,11 @@ require (
 	github.com/dana-team/nfspvc-operator v0.2.2
 	github.com/go-logr/logr v1.4.1
 	github.com/go-logr/zapr v1.3.0
-	github.com/kube-logging/logging-operator/pkg/sdk v0.11.0
+	github.com/kube-logging/logging-operator/pkg/sdk v0.11.1-0.20240314152935-421fefebc813
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.31.1
 	github.com/openshift/api v0.0.0-20240410141538-3c0461467316
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	go.elastic.co/ecszap v1.0.2
 	go.uber.org/zap v1.27.0
 	golang.org/x/net v0.24.0
@@ -29,7 +29,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blendle/zapdriver v1.3.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
@@ -57,8 +57,8 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0 // indirect
 	github.com/prometheus/client_golang v1.19.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.52.2 // indirect

diff --git a/go.sum b/go.sum
@@ -24,8 +24,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/dana-team/nfspvc-operator v0.2.2 h1:/FwVD34O39ky9lQyOiVsyv8Et7K0Wkai6kseDThmR2o=
 github.com/dana-team/nfspvc-operator v0.2.2/go.mod h1:l4J/LPQPf2ksPDZJu3RO3k4srKnXJ3fCS3JG0XTYZ5o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
@@ -99,8 +100,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kube-logging/logging-operator/pkg/sdk v0.11.0 h1:3yg+fdk6LnS+wVThEOr85qTtTKmS+6e5DrCtG1j6yFA=
-github.com/kube-logging/logging-operator/pkg/sdk v0.11.0/go.mod h1:dWCUhDTFiW5V21LQJ2hoFzqZb1y6U1mFQU7ZXzd9tFE=
+github.com/kube-logging/logging-operator/pkg/sdk v0.11.1-0.20240314152935-421fefebc813 h1:OMV8NhQGJYD2XQe5V1elb6USyp7XwdugCUIc34+o8G4=
+github.com/kube-logging/logging-operator/pkg/sdk v0.11.1-0.20240314152935-421fefebc813/go.mod h1:KyAHHsYc1nA6ImWIYYZwqlw8KlalpHCwjOUqd+iv8N8=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -125,10 +126,11 @@ github.com/openshift/api v0.0.0-20240410141538-3c0461467316/go.mod h1:CxgbWAlvu2
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 h1:yl9ceUSUBo9woQIO+8eoWpcxZkdZgm89g+rVvu37TUw=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0/go.mod h1:9Uuu3pEU2jB8PwuqkHvegQ0HV/BlZRJUyfTYAqfdVF8=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0 h1:9h7PxMhT1S8lOdadEKJnBh3ELMdO60XkoDV98grYjuM=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0/go.mod h1:4FiLCL664L4dNGeqZewiiD0NS7hhqi/CxyM4UOq5dfM=
 github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
 github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
@@ -156,8 +158,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=

diff --git a/hack/kind-config.yaml b/hack/kind-config.yaml