diff --git a/Makefile b/Makefile index bdc724df..2ec75320 100644 --- a/Makefile +++ b/Makefile @@ -167,7 +167,8 @@ enable-nfs-knative: ## Enable NFS for Knative .PHONY: install-logging install-logging: ## Install logging operator on the kind cluster - helm upgrade --install --wait --create-namespace --namespace logging logging-operator oci://ghcr.io/kube-logging/helm-charts/logging-operator --set logging.enabled=true + helm upgrade --install --wait --create-namespace --namespace logging-operator-system logging-operator oci://ghcr.io/kube-logging/helm-charts/logging-operator + kubectl apply -f hack/logging-operator-resources.yaml KNATIVE_URL ?= https://github.com/knative-extensions/kn-plugin-quickstart/releases/download/knative-v1.11.2/kn-quickstart-linux-amd64 KNATIVE_HPA_URL ?= https://github.com/knative/serving/releases/download/knative-v1.11.2/serving-hpa.yaml diff --git a/README.md b/README.md index dea7fcd9..5728d941 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The `container-app-operator` project can work as a standalone solution, but is m - [x] Support for autoscaler (`HPA` or `KPA`) according to the chosen `scaleMetric` (`concurrency`, `rps`, `cpu`, `memory`) with default settings. - [x] Support for HTTP/HTTPS `DomainMapping` for accessing applications via `Ingress`/`Route`. - [x] Support for all `Knative Serving` configurations. -- [x] Support for exporting logs to `Elasticsearch` and `Splunk` indexes. +- [x] Support for exporting logs to an `Elasticsearch` index. - [x] Support for changing the state of `Capp` from `enabled` (workload is in running state) to `disabled` (workload is not in running state). - [x] Support for external NFS storage connected to `Capp` by using `volumeMounts`. @@ -136,9 +136,8 @@ spec: type: elastic host: 10.11.12.13 index: main - username: elastic - passwordSecretName: es-elastic-user - sslVerify: false + user: elastic + passwordSecret: es-elastic-user scaleMetric: concurrency state: enabled ``` diff --git a/api/v1alpha1/capp_types.go b/api/v1alpha1/capp_types.go index d534e916..c2038279 100755 --- a/api/v1alpha1/capp_types.go +++ b/api/v1alpha1/capp_types.go @@ -107,7 +107,8 @@ type RouteSpec struct { // LogSpec defines the configuration for shipping Capp logs. type LogSpec struct { // Type defines where to send the Capp logs - // Possible values : "elastic" and "splunk". + // +kubebuilder:default:="elastic" + // +kubebuilder:validation:Enum=elastic // +optional Type string `json:"type,omitempty"` @@ -115,27 +116,18 @@ type LogSpec struct { // +optional Host string `json:"host,omitempty"` - // SSLVerify determines whether to skip ssl verification. - // +optional - SSLVerify bool `json:"sslVerify,omitempty"` - // Index defines the index name to write events to. // +optional Index string `json:"index,omitempty"` - // UserName defines a User for authentication. + // User defines a User for authentication. // +optional - UserName string `json:"username,omitempty"` + User string `json:"username,omitempty"` - // PasswordSecretName defines the name of the secret + // PasswordSecret defines the name of the secret // containing the password for authentication. // +optional - PasswordSecretName string `json:"passwordSecretName,omitempty"` - - // HecTokenSecretName defines the name of the secret - // containing the Splunk Hec token. - // +optional - HecTokenSecretName string `json:"hecTokenSecretName,omitempty"` + PasswordSecret string `json:"passwordSecretName,omitempty"` } // ApplicationLinks contains relevant information about @@ -171,17 +163,17 @@ type StateStatus struct { LastChange metav1.Time `json:"lastChange,omitempty"` } -// LoggingStatus defines the state of the Flow and Output objects linked to the Capp. +// LoggingStatus defines the state of the SyslogNGFlow and SyslogNGOutput objects linked to the Capp. type LoggingStatus struct { - // Flow represents the Status of the Flow used by the Capp. + // SyslogNGFlow represents the Status of the SyslogNGFlow used by the Capp. // +optional - Flow loggingv1beta1.FlowStatus `json:"flow,omitempty"` + SyslogNGFlow loggingv1beta1.SyslogNGFlowStatus `json:"syslogngflow,omitempty"` - // Output represents the Status of the Output used by the Capp. + // SyslogNGOutput represents the Status of the SyslogNGOutput used by the Capp. // +optional - Output loggingv1beta1.OutputStatus `json:"output,omitempty"` + SyslogNGOutput loggingv1beta1.SyslogNGOutputStatus `json:"syslogngoutput,omitempty"` - // Conditions contain details about the current state of the Output and Flow used by the Capp. + // Conditions contain details about the current state of the SyslogNGFlow and SyslogNGOutput used by the Capp. // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c680a5f3..6d5f3c14 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -172,8 +172,8 @@ func (in *LogSpec) DeepCopy() *LogSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoggingStatus) DeepCopyInto(out *LoggingStatus) { *out = *in - in.Flow.DeepCopyInto(&out.Flow) - in.Output.DeepCopyInto(&out.Output) + in.SyslogNGFlow.DeepCopyInto(&out.SyslogNGFlow) + in.SyslogNGOutput.DeepCopyInto(&out.SyslogNGOutput) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/config/crd/bases/rcs.dana.io_capps.yaml b/config/crd/bases/rcs.dana.io_capps.yaml index 94dbabb6..0e15d273 100644 --- a/config/crd/bases/rcs.dana.io_capps.yaml +++ b/config/crd/bases/rcs.dana.io_capps.yaml @@ -7642,11 +7642,6 @@ spec: logSpec: description: LogSpec defines the configuration for shipping Capp logs. properties: - hecTokenSecretName: - description: |- - HecTokenSecretName defines the name of the secret - containing the Splunk Hec token. - type: string host: description: Host defines Elasticsearch or Splunk host. type: string @@ -7655,19 +7650,17 @@ spec: type: string passwordSecretName: description: |- - PasswordSecretName defines the name of the secret + PasswordSecret defines the name of the secret containing the password for authentication. type: string - sslVerify: - description: SSLVerify determines whether to skip ssl verification. - type: boolean type: - description: |- - Type defines where to send the Capp logs - Possible values : "elastic" and "splunk". + default: elastic + description: Type defines where to send the Capp logs + enum: + - elastic type: string username: - description: UserName defines a User for authentication. + description: User defines a User for authentication. type: string type: object routeSpec: @@ -8165,7 +8158,7 @@ spec: properties: conditions: description: Conditions contain details about the current state - of the Output and Flow used by the Capp. + of the SyslogNGFlow and SyslogNGOutput used by the Capp. items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended @@ -8236,9 +8229,9 @@ spec: - type type: object type: array - flow: - description: Flow represents the Status of the Flow used by the - Capp. + syslogngflow: + description: SyslogNGFlow represents the Status of the SyslogNGFlow + used by the Capp. properties: active: type: boolean @@ -8249,9 +8242,9 @@ spec: problemsCount: type: integer type: object - output: - description: Output represents the Status of the Output used by - the Capp. + syslogngoutput: + description: SyslogNGOutput represents the Status of the SyslogNGOutput + used by the Capp. properties: active: type: boolean diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84..ad13e96b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 40e7394e..990f2fe0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -53,7 +53,7 @@ rules: - apiGroups: - logging.banzaicloud.io resources: - - flows + - syslogngflows verbs: - create - delete @@ -64,7 +64,7 @@ rules: - apiGroups: - logging.banzaicloud.io resources: - - outputs + - syslogngoutputs verbs: - create - delete diff --git a/go.mod b/go.mod index 8e732c36..da0f618f 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,11 @@ require ( github.com/dana-team/nfspvc-operator v0.2.2 github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 - github.com/kube-logging/logging-operator/pkg/sdk v0.11.0 + github.com/kube-logging/logging-operator/pkg/sdk v0.11.1-0.20240314152935-421fefebc813 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.31.1 github.com/openshift/api v0.0.0-20240410141538-3c0461467316 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.elastic.co/ecszap v1.0.2 go.uber.org/zap v1.27.0 golang.org/x/net v0.24.0 @@ -29,7 +29,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -57,8 +57,8 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0 // indirect github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.52.2 // indirect diff --git a/go.sum b/go.sum index 64a5688d..ea887ba4 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/dana-team/nfspvc-operator v0.2.2 h1:/FwVD34O39ky9lQyOiVsyv8Et7K0Wkai6kseDThmR2o= github.com/dana-team/nfspvc-operator v0.2.2/go.mod h1:l4J/LPQPf2ksPDZJu3RO3k4srKnXJ3fCS3JG0XTYZ5o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= @@ -99,8 +100,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kube-logging/logging-operator/pkg/sdk v0.11.0 h1:3yg+fdk6LnS+wVThEOr85qTtTKmS+6e5DrCtG1j6yFA= -github.com/kube-logging/logging-operator/pkg/sdk v0.11.0/go.mod h1:dWCUhDTFiW5V21LQJ2hoFzqZb1y6U1mFQU7ZXzd9tFE= +github.com/kube-logging/logging-operator/pkg/sdk v0.11.1-0.20240314152935-421fefebc813 h1:OMV8NhQGJYD2XQe5V1elb6USyp7XwdugCUIc34+o8G4= +github.com/kube-logging/logging-operator/pkg/sdk v0.11.1-0.20240314152935-421fefebc813/go.mod h1:KyAHHsYc1nA6ImWIYYZwqlw8KlalpHCwjOUqd+iv8N8= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -125,10 +126,11 @@ github.com/openshift/api v0.0.0-20240410141538-3c0461467316/go.mod h1:CxgbWAlvu2 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 h1:yl9ceUSUBo9woQIO+8eoWpcxZkdZgm89g+rVvu37TUw= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0/go.mod h1:9Uuu3pEU2jB8PwuqkHvegQ0HV/BlZRJUyfTYAqfdVF8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0 h1:9h7PxMhT1S8lOdadEKJnBh3ELMdO60XkoDV98grYjuM= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0/go.mod h1:4FiLCL664L4dNGeqZewiiD0NS7hhqi/CxyM4UOq5dfM= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= @@ -156,8 +158,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/hack/kind-config.yaml b/hack/kind-config.yaml deleted file mode 100644 index 658ea43e..00000000 --- a/hack/kind-config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# four node (one control plane + three workers) cluster config for k8s e2e test in github workflow! -# for the local registry config see https://kind.sigs.k8s.io/docs/user/local-registry/ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - - role: worker - - role: worker - - role: worker -containerdConfigPatches: - - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."kind-registry:5000"] - endpoint = ["http://kind-registry:5000"] \ No newline at end of file diff --git a/hack/logging-operator-resources.yaml b/hack/logging-operator-resources.yaml new file mode 100644 index 00000000..013330ad --- /dev/null +++ b/hack/logging-operator-resources.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: logging +spec: + clusterDomain: cluster.local + configCheck: + timeoutSeconds: 10 + controlNamespace: logging-operator-system +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: FluentbitAgent +metadata: + name: fluentbit-agent + namespace: logging-operator-system +spec: + filterKubernetes: + Annotations: 'Off' + inputTail: + Parser: cri-log-key + customParsers: | + [PARSER] + Name cri-log-key + Format regex + Regex ^(?