diff --git a/.gitignore b/.gitignore index 7f02333d..e3060dcf 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,11 @@ Dockerfile.cross *.swp *.swo *~ + +crddir + +.licensei.cache + +.DS_Store + +go.work.sum \ No newline at end of file diff --git a/Makefile b/Makefile index b87f42e4..6a7f3d0d 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,11 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./api/..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." .PHONY: fmt fmt: ## Run go fmt against code. @@ -190,8 +190,14 @@ $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +ENVTEST_OTEL_OPERATOR_VERSION=v0.94.0 +# Download CRDs for envtest +crddir/github.com/open-telemetry/opentelemetry-operator: + git clone --depth 1 --branch ${ENVTEST_OTEL_OPERATOR_VERSION} https://github.com/open-telemetry/opentelemetry-operator.git crddir/github.com/open-telemetry/opentelemetry-operator + .PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +envtest: $(ENVTEST) crddir/github.com/open-telemetry/opentelemetry-operator ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest @@ -264,3 +270,4 @@ find ${BIN} -name '$(notdir ${IMPORT_PATH})_*' -exec rm {} + GOBIN=${BIN} go install ${IMPORT_PATH}@${VERSION} mv ${BIN}/$(notdir ${IMPORT_PATH}) $@ endef + diff --git a/api/telemetry/v1alpha1/zz_generated.deepcopy.go b/api/telemetry/v1alpha1/zz_generated.deepcopy.go index d7a784dc..3dd59244 100644 --- a/api/telemetry/v1alpha1/zz_generated.deepcopy.go +++ b/api/telemetry/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ //go:build !ignore_autogenerated -// Copyright © 2023 Kube logging authors +// Copyright © 2024 Kube logging authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/go.mod b/go.mod index e1ba642f..0ed355d2 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect go.opentelemetry.io/collector/featuregate v0.77.0 // indirect + logur.dev/logur v0.17.0 // indirect ) require ( @@ -91,6 +92,7 @@ require ( k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect + logur.dev/integration/logr v0.5.0 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 01edf67c..af640f18 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,7 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -272,6 +273,10 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +logur.dev/integration/logr v0.5.0 h1:Xy4/PIBbGKKsKGt+86sBJhMa8MSX5+XsyOaoBBU+tnw= +logur.dev/integration/logr v0.5.0/go.mod h1:Kp0WOJ6LRV7ewobPZddHHoLmLOpDmi+phZFhpUWD1Qg= +logur.dev/logur v0.17.0 h1:lwFZk349ZBY7KhonJFLshP/VhfFa6BxOjHxNnPHnEyc= +logur.dev/logur v0.17.0/go.mod h1:DyA5B+b6WjjCcnpE1+HGtTLh2lXooxRq+JmAwXMRK08= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 9ce3133f..40c40cb7 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,4 +1,4 @@ -// Copyright © 2023 Kube logging authors +// Copyright © 2024 Kube logging authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/controller/telemetry/collector_controller.go b/internal/controller/telemetry/collector_controller.go index f4d09c67..0f3f9e62 100644 --- a/internal/controller/telemetry/collector_controller.go +++ b/internal/controller/telemetry/collector_controller.go @@ -209,10 +209,11 @@ func (r *CollectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( Namespace: collector.Spec.ControlNamespace, }, Spec: otelv1alpha1.OpenTelemetryCollectorSpec{ - Config: otelConfig, - Mode: otelv1alpha1.ModeDaemonSet, - Image: "ghcr.io/axoflow/axoflow-otel-collector/axoflow-otel-collector:0.94.0", - ServiceAccount: saName.Name, + UpgradeStrategy: "none", + Config: otelConfig, + Mode: otelv1alpha1.ModeDaemonSet, + Image: "ghcr.io/axoflow/axoflow-otel-collector/axoflow-otel-collector:0.94.0", + ServiceAccount: saName.Name, VolumeMounts: []apiv1.VolumeMount{ { Name: "varlog", diff --git a/internal/controller/telemetry/controller_integration_test.go b/internal/controller/telemetry/controller_integration_test.go new file mode 100644 index 00000000..d8b2ab2e --- /dev/null +++ b/internal/controller/telemetry/controller_integration_test.go @@ -0,0 +1,299 @@ +// Copyright © 2024 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + "os" + "path" + "time" + + "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + otelv1alpha1 "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/yaml" + //+kubebuilder:scaffold:imports +) + +var _ = Describe("Telemetry controller integration test", func() { + const ( + timeout = time.Second * 5 + interval = time.Millisecond * 250 + ) + + Context("Deploying a telemetry pipeline", Ordered, func() { + It("Namespace should exist beforehand ", func() { + namespaces := []v1.Namespace{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-1-workload", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-1-ctrl", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-2-all", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "collector", + }, + }, + } + + for _, namespace := range namespaces { + Expect(k8sClient.Create(ctx, &namespace)).Should(Succeed()) + } + + }) + + It("Subscriptions should be created in the annotated namespaces", func() { + ctx := context.Background() + + subscriptions := []v1alpha1.Subscription{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "subscription-example-1", + Namespace: "tenant-1-ctrl", + }, + Spec: v1alpha1.SubscriptionSpec{ + OTTL: "route()", + Outputs: []v1alpha1.NamespacedName{ + { + Name: "otlp-test-output-1", + Namespace: "collector", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "subscription-example-2", + Namespace: "tenant-2-all", + }, + Spec: v1alpha1.SubscriptionSpec{ + OTTL: "route()", + Outputs: []v1alpha1.NamespacedName{ + { + Name: "otlp-test-output-2", + Namespace: "collector", + }, + }, + }, + }, + } + + for _, subscription := range subscriptions { + Expect(k8sClient.Create(ctx, &subscription)).Should(Succeed()) + } + }) + + It("Tenants should be created", func() { + tenants := []v1alpha1.Tenant{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-1", + }, + Spec: v1alpha1.TenantSpec{ + SubscriptionNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": "tenant-1-ctrl", + }, + }, + }, + LogSourceNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": "tenant-1-workload", + }, + }, + }, + }, + Status: v1alpha1.TenantStatus{ + Subscriptions: []string{"asd", "bsd"}, + LogSourceNamespaces: []string{}, + Collector: "asd", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-2", + }, + Spec: v1alpha1.TenantSpec{ + SubscriptionNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": "tenant-2-all", + }, + }, + }, + LogSourceNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": "tenant-2-all", + }, + }, + }, + }, + Status: v1alpha1.TenantStatus{ + Subscriptions: []string{"asd", "bsd"}, + LogSourceNamespaces: []string{}, + Collector: "asd", + }, + }, + } + + for _, tenant := range tenants { + Expect(k8sClient.Create(ctx, &tenant)).Should(Succeed()) + } + }) + + It("Outputs should be created", func() { + + outputs := []v1alpha1.OtelOutput{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "otlp-test-output", + Namespace: "collector", + }, + Spec: v1alpha1.OtelOutputSpec{ + OTLP: v1alpha1.OTLPgrpc{ + ClientConfig: v1alpha1.ClientConfig{ + Endpoint: "receiver-collector.example-tenant-ns.svc.cluster.local:4317", + TLSSetting: v1alpha1.TLSClientSetting{ + Insecure: true, + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "otlp-test-output-2", + Namespace: "collector", + }, + Spec: v1alpha1.OtelOutputSpec{ + OTLP: v1alpha1.OTLPgrpc{ + ClientConfig: v1alpha1.ClientConfig{ + Endpoint: "receiver-collector.example-tenant-ns.svc.cluster.local:4317", + TLSSetting: v1alpha1.TLSClientSetting{ + Insecure: true, + }, + }, + }, + }, + }, + } + + for _, output := range outputs { + Expect(k8sClient.Create(ctx, &output)).Should(Succeed()) + } + }) + + It("Collector should be created", func() { + collector := v1alpha1.Collector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-collector", + }, + Spec: v1alpha1.CollectorSpec{ + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{}, + MatchExpressions: []metav1.LabelSelectorRequirement{}, + }, + ControlNamespace: "collector", + }, + } + Expect(k8sClient.Create(ctx, &collector)).Should(Succeed()) + }) + + }) + When("The controller reconciles based on deployed resources", Ordered, func() { + + It("RBAC resources should be reconciled by controller", func() { + + createdServiceAccount := &v1.ServiceAccount{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: "collector", Name: "example-collector-sa"}, createdServiceAccount) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(createdServiceAccount.OwnerReferences[0].Name).To(Equal("example-collector")) + + createdClusterRole := &rbacv1.ClusterRole{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: "example-collector-pod-association-reader"}, createdClusterRole) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(createdClusterRole.OwnerReferences[0].Name).To(Equal("example-collector")) + + createdClusterRoleBinding := &rbacv1.ClusterRoleBinding{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: "example-collector-crb"}, createdClusterRoleBinding) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(createdClusterRoleBinding.OwnerReferences[0].Name).To(Equal("example-collector")) + + }) + + It("OpentelemetryCollector resource should be reconciled by controller", func() { + + createdOtelCollector := &otelv1alpha1.OpenTelemetryCollector{} + + expectedConfig, err := os.ReadFile(path.Join("envtest_testdata", "config.yaml")) + Expect(err).NotTo(HaveOccurred()) + exptectedOtelCollector := &otelv1alpha1.OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "otelcollector-example-collector", + Namespace: "collector", + }, + Spec: otelv1alpha1.OpenTelemetryCollectorSpec{ + Config: string(expectedConfig), + }, + } + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: "collector", Name: "otelcollector-example-collector"}, createdOtelCollector) + return err == nil + }, timeout, interval).Should(BeTrue()) + + expectedMap := map[string]any{} + err = yaml.Unmarshal([]byte(exptectedOtelCollector.Spec.Config), expectedMap) + Expect(err).NotTo(HaveOccurred()) + + createdMap := map[string]any{} + err = yaml.Unmarshal([]byte(createdOtelCollector.Spec.Config), createdMap) + Expect(err).NotTo(HaveOccurred()) + + Expect(createdMap).To(Equal(expectedMap)) + + }) + }) +}) diff --git a/internal/controller/telemetry/envtest_testdata/config.yaml b/internal/controller/telemetry/envtest_testdata/config.yaml new file mode 100644 index 00000000..af2875b4 --- /dev/null +++ b/internal/controller/telemetry/envtest_testdata/config.yaml @@ -0,0 +1,137 @@ +receivers: + filelog/kubernetes: + exclude: + - /var/log/pods/*/otc-container/*.log + include: + - /var/log/pods/*/*/*.log + include_file_name: false + include_file_path: true + operators: + - id: get-format + routes: + - expr: body matches "^\\{" + output: parser-docker + - expr: body matches "^[^ Z]+Z" + output: parser-containerd + type: router + - id: parser-containerd + output: extract_metadata_from_filepath + regex: ^(?P