fix: e2e-tests ran on feature images

Signed-off-by: Bence Csati <bence.csati@axoflow.com>
kube-logging · Oct 31, 2024 · f5b9c19 · f5b9c19
1 parent 35aabc6
commit f5b9c19
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 135 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -8,8 +8,7 @@ on:
   pull_request:
 
 env:
-  GO_VERSION: '1.22.1'
-  KUBECTL_VERSION: 'v1.24.1'
+  KUBECTL_VERSION: 'v1.26.6'
 
 jobs:
   build:
@@ -26,7 +25,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          tags: controller:latest
+          tags: controller:local
           cache-from: type=gha
           cache-to: type=gha,mode=max
           outputs: type=docker,dest=/tmp/controller.tar
@@ -37,66 +36,14 @@ jobs:
           name: controller
           path: /tmp/controller.tar
 
-  go:
-    name: Go end2end tests
-    runs-on: ubuntu-latest
-    needs: build
-    strategy:
-      fail-fast: false
-#      matrix:
-#        SHARD: [0]
-#        SHARDS: [1]
-
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: controller
-          path: /tmp
-
-      - name: Load image
-        run: |
-          docker load --input /tmp/controller.tar
-          docker image ls -a
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Set up kubectl
-        uses: azure/setup-kubectl@v4.0.0
-        with:
-          version: ${{ env.KUBECTL_VERSION }}
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Run e2e tests
-        run: make e2e-test-ci
-#        env:
-#          SHARD: ${{ matrix.SHARD }}
-#          SHARDS: ${{ matrix.SHARDS }}
-
-      - name: Archive Test Results
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: go-e2e-test-cluster-logs
-          path: build/_test
-          retention-days: 5
-
-  e2e-test:
-    name: Shell script tests with different k8s versions
+  chart:
+    name: Shell script tests with chart install
     runs-on: ubuntu-latest
     needs: build
     strategy:
       fail-fast: false
       matrix:
-        kube: ["1.26", "1.27", "1.28", "1.29"]
+        kube: ["1.29", "1.30", "1.31"]
 
     steps:
       - name: Set up Docker Buildx
@@ -118,11 +65,6 @@ jobs:
         with:
           version: ${{ env.KUBECTL_VERSION }}
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
       - name: Checkout code
         uses: actions/checkout@v4
 
@@ -131,14 +73,12 @@ jobs:
         id: node_image
         run: |
           case ${{ matrix.kube }} in
-            1.26)
-              NODE_IMAGE=kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb ;;
-            1.27)
-              NODE_IMAGE=kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72 ;;
-            1.28)
-              NODE_IMAGE=kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31 ;;
             1.29)
-              NODE_IMAGE=kindest/node:v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570 ;;
+              NODE_IMAGE=kindest/node:v1.29.4@sha256:ea40a6bd365a17f71fd3883a1d34a0791d7d6b0eb75832c6d85b6f2326827f1e ;;
+            1.30)
+              NODE_IMAGE=kindest/node:v1.30.0@sha256:2af5d1b382926abcd6336312d652cd045b7cc47475844a608669c71b1fefcfbc ;;
+            1.31)
+              NODE_IMAGE=kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865 ;;
             esac
 
             echo "image=$NODE_IMAGE" >> $GITHUB_OUTPUT
@@ -148,8 +88,8 @@ jobs:
         env:
           KIND_IMAGE: ${{ steps.node_image.outputs.image }}
 
-      - name: Test script for E2E
-        run: make e2e-test-ci
+      - name: Test
+        run: e2e/e2e_test.sh
 
       - name: Print last 10k kubernetes logs from default, collector and example-tenant-ns namespaces
         if: always()

diff --git a/Makefile b/Makefile
@@ -5,17 +5,15 @@ export PATH := $(BIN):$(PATH)
 GOVERSION := $(shell go env GOVERSION)
 
 KIND := ${BIN}/kind
-KIND_VERSION ?= v0.20.0
-KIND_IMAGE ?= kindest/node:v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570
-KIND_CLUSTER ?= kind
-
-CI_MODE_ENABLED := ""
-NO_KIND_CLEANUP := ""
+KIND_VERSION ?= v0.24.0
+KIND_IMAGE ?= kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865
+KIND_CLUSTER := kind
 
 # Image URL to use all building/pushing image targets
 IMG ?= controller:local
+
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.28.0
+ENVTEST_K8S_VERSION = 1.31.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -24,9 +22,6 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-TIMEOUT_CMD=timeout
-
-
 # CONTAINER_TOOL defines the container tool to be used for building images.
 # Be aware that the target commands are only tested with Docker which is
 # scaffolded by default. However, you might want to replace it to use other
@@ -39,7 +34,7 @@ SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
 LICENSEI := ${BIN}/licensei
-LICENSEI_VERSION = v0.8.0
+LICENSEI_VERSION = v0.9.0
 
 ##@ General
 
@@ -88,7 +83,7 @@ test: manifests generate fmt vet envtest ## Run verifications and tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./... -coverprofile cover.out
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
-GOLANGCI_LINT_VERSION ?= v1.55.2
+GOLANGCI_LINT_VERSION ?= v1.61.0
 golangci-lint:
 	@[ -f $(GOLANGCI_LINT) ] || { \
 	set -e ;\
@@ -108,15 +103,9 @@ run-delve: generate fmt vet manifests
 	go build -o bin/manager cmd/main.go
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./bin/manager
 
-.PHONY: e2e-test
-e2e-test: ## Run e2e tests
-	cd e2e && export CI_MODE=$(CI_MODE_ENABLED) NO_KIND_CLEANUP=$(NO_KIND_CLEANUP) && $(TIMEOUT_CMD) --foreground 15m ./e2e_test.sh || (echo "E2E test failed"; exit 1)
-
-.PHONY: e2e-test-ci
-e2e-test-ci: CI_MODE_ENABLED=1
-e2e-test-ci: NO_KIND_CLEANUP=1
-e2e-test-ci: IMG="controller:latest" ## Run e2e tests, telemetry collector runs inside k8s
-e2e-test-ci: docker-build e2e-test
+.PHONY: test-e2e
+test-e2e: docker-build kind-cluster ## Run e2e tests
+	e2e/e2e_test.sh || (echo "E2E test failed"; exit 1)
 
 .PHONY: check-diff
 check-diff: generate
@@ -211,7 +200,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.2.1
-CONTROLLER_TOOLS_VERSION ?= v0.14.0
+CONTROLLER_TOOLS_VERSION ?= v0.16.3
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading.
@@ -228,7 +217,6 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 	test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \
 	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
 
-
 ENVTEST_OTEL_OPERATOR_VERSION=v0.103.0
 # Download CRDs for envtest
 crddir/github.com/open-telemetry/opentelemetry-operator:

diff --git a/e2e/e2e_test.sh b/e2e/e2e_test.sh
@@ -3,58 +3,92 @@
 set -eou pipefail
 set -o xtrace
 
-create_if_does_not_exist() {
-  local resource_type=$1
-  local resource_name=$2
-  kubectl create  "${resource_type}" "${resource_name}" --dry-run=client -o yaml | kubectl apply -f-
-}
+function main()
+{
+    kubectl get namespace telemetry-controller-system || kubectl create namespace telemetry-controller-system
+    kubectl config set-context --current --namespace=telemetry-controller-system
 
-KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_E2E:-so-e2e}
-NO_KIND_CLEANUP=${NO_KIND_CLEANUP:-}
-CI_MODE=${CI_MODE:-}
-  # Backup current kubernetes context
-CURRENT_K8S_CTX=$(kubectl config view | grep "current" | cut -f 2 -d : | xargs)
+    load_images
 
-TIMEOUT_CMD=timeout
+    helm_deploy_telemetry_controller
 
+    helm_deploy_log_generator
 
-# HELM BASED DEPLOYMENT
+    deploy_test_assets
 
-# Prepare env
-kind create cluster --name "${KIND_CLUSTER_NAME}" --wait 5m
-kubectl config set-context kind-"${KIND_CLUSTER_NAME}"
+    # Check for received messages - subscription-sample-1
+    # NOTE: We should not use grep -q, because it causes a SIGPIPE for kubectl and we have -o pipefail
+    check_subscription_logs "subscription-sample-1"
+    check_subscription_logs "subscription-sample-2"
 
-# Install telemetry-controller and opentelemetry-operator
-helm upgrade --install --wait --create-namespace --namespace telemetry-controller-system telemetry-controller oci://ghcr.io/kube-logging/helm-charts/telemetry-controller
+    echo "E2E (helm) test: PASSED"
+}
 
-# Wait for the pod to be ready, without it the webhook backend service will be unavailable.
-sleep 10
+function load_images()
+{
+    local images=("controller:local")
+    for image in "${images[@]}"; do
+        kind load docker-image "${image}"
+    done
+}
 
-# Use example
-kubectl apply -f ../e2e/testdata/one_tenant_two_subscriptions
+function helm_deploy_telemetry_controller()
+{
+    helm upgrade --install \
+        --debug \
+        --wait \
+        --create-namespace \
+        -f e2e/values.yaml \
+        telemetry-controller \
+        "charts/telemetry-controller/"
+}
 
-# Create log-generator
-helm install --wait --create-namespace --namespace example-tenant-ns --generate-name oci://ghcr.io/kube-logging/helm-charts/log-generator
+function helm_deploy_log_generator()
+{
+    helm install \
+        --wait \
+        --create-namespace \
+        --namespace example-tenant-ns \
+        --generate-name \
+        oci://ghcr.io/kube-logging/helm-charts/log-generator
+}
 
+function deploy_test_assets()
+{
+    kubectl apply -f e2e/testdata/one_tenant_two_subscriptions/
 
-# Check for received messages - subscription-sample-1
-# NOTE: We should not use grep -q, because it causes a SIGPIPE for kubectl and we have -o pipefail
-echo "Checking for subscription-sample-1 in deployments/receiver-collector logs"
-while
-  ! kubectl logs --namespace example-tenant-ns deployments/receiver-collector |  grep "subscription-sample-1"
-
-do true; done
+    sleep 5
 
-# Check for received messages - subscription-sample-2
-echo "Checking for subscription-sample-2 in deployments/receiver-collector logs"
-while
-  ! kubectl logs --namespace example-tenant-ns deployments/receiver-collector |  grep "subscription-sample-2"
+    # Wait for the deployment to be ready
+    kubectl wait --for=condition=available --timeout=300s deployment/receiver-collector --namespace example-tenant-ns
+}
 
-do true; done
+function check_subscription_logs()
+{
+    local subscription="$1"
+    local namespace="${2:-example-tenant-ns}"
+    local deployment="${3:-receiver-collector}"
+    local max_duration=300
+    local start_time=$(date +%s)
 
-echo "E2E (helm) test: PASSED"
+    echo "Checking for $subscription in $namespace/$deployment logs"
+
+    while true; do
+        if kubectl logs --namespace "$namespace" "deployments/$deployment" | grep -E "$subscription"; then
+            echo "Found $subscription in logs"
+            return 0
+        fi
+
+        sleep 1
+
+        local current_time=$(date +%s)
+        local elapsed_time=$((current_time - start_time))
+
+        if [ "$elapsed_time" -ge "$max_duration" ]; then
+            echo "ERROR: Subscription $subscription not found in logs after $max_duration seconds"
+            return 1
+        fi
+    done
+}
 
-# Check if cluster should be removed, ctx restored
-if [[ -z "${NO_KIND_CLEANUP}" ]]; then
-  kind delete cluster --name "${KIND_CLUSTER_NAME}"
-fi
+main "$@"
diff --git a/e2e/values.yaml b/e2e/values.yaml
@@ -0,0 +1,3 @@
+image:
+  repository: controller
+  tag: local