From a9efa3ea2d966568cf7b795835449a04ab3cabd4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 2 May 2024 18:38:23 +0200 Subject: [PATCH] [Buildkite] Add pipeline to test with Elastic serverless daily (#1807) Add pipeline to test with an Elastic serverless project daily. It also adds the required settings to trigger this pipeline from a Pull Request. --- .buildkite/hooks/pre-command | 39 +++++- .buildkite/hooks/pre-exit | 11 +- .buildkite/pipeline.serverless.yml | 60 +++++++++ .buildkite/pull-requests.json | 16 +++ .buildkite/scripts/integration_tests.sh | 75 ++++------- .buildkite/scripts/release.sh | 7 +- .buildkite/scripts/test-with-integrations.sh | 22 +--- .../scripts/test_packages_with_serverless.sh | 87 +++++++++++++ .buildkite/scripts/tooling.sh | 118 +++++++++++++++++- README.md | 20 +++ catalog-info.yaml | 51 ++++++++ scripts/test-check-packages.sh | 44 +++++-- tools/readme/readme.md.tmpl | 20 +++ 13 files changed, 480 insertions(+), 90 deletions(-) create mode 100644 .buildkite/pipeline.serverless.yml create mode 100644 .buildkite/scripts/test_packages_with_serverless.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b05154255..7255f62d9 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,11 +6,18 @@ set -euo pipefail GO_VERSION=$(cat .go-version) export GO_VERSION +export SERVERLESS=${SERVERLESS:-"false"} +export WORKSPACE=$(pwd) + + GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account +EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa +EC_DATA_PATH=secret/ci/elastic-elastic-package/ec_data + # variables required for terraform export ENVIRONMENT="ci" REPO=$(repo_name "${BUILDKITE_REPO}") @@ -46,12 +53,24 @@ export CREATED_DATE # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables +export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}" +export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" + is_step_required_to_upload_safe_logs() { - if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" && "$BUILDKITE_PIPELINE_SLUG" != "elastic-package-test-serverless" ]]; then return 1 fi - if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then - return 0 + + if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" ]]; then + if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then + return 0 + fi + fi + + if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-serverless" ]]; then + if [[ "$BUILDKITE_STEP_KEY" == "test-serverless" ]]; then + return 0 + fi fi return 1 } @@ -139,3 +158,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" && "$BUILDKI export ELASTIC_PACKAGE_GCP_EMAIL_SECRET fi + +if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" ]]; then + if [[ "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then + EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}") + export EC_API_KEY_SECRET + EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}") + export EC_HOST_SECRET + EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}") + export EC_REGION_SECRET + + GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) + export GITHUB_TOKEN + fi +fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 591b7acfe..92f0f4d7d 100755 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -4,6 +4,16 @@ source .buildkite/scripts/tooling.sh set -euo pipefail +if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then + echo "--- Take down the Elastic stack" + # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package + # is already installed in the test_serverless pipeline step, accessing + # directly to the binary + EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package stack down -v +fi + +echo "--- Cleanup" +cleanup unset_secrets # integrations-parallel-gcp @@ -15,4 +25,3 @@ unset ELASTIC_PACKAGE_AWS_ACCESS_KEY unset ELASTIC_PACKAGE_AWS_SECRET_KEY unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY - diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml new file mode 100644 index 000000000..83c537e35 --- /dev/null +++ b/.buildkite/pipeline.serverless.yml @@ -0,0 +1,60 @@ +env: + NOTIFY_TO: "ecosystem-team@elastic.co" + SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + DOCKER_COMPOSE_VERSION: "v2.24.1" + DOCKER_VERSION: "26.1.0" + KIND_VERSION: 'v0.20.0' + K8S_VERSION: 'v1.29.0' + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + GH_CLI_VERSION: "2.29.0" + # Elastic package settings + # Manage docker output/logs + ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true" + # Default license to use by `elastic-package build` + ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt" + # Link definitions path (full path to be set in the corresponding step) + ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml" + # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields + ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true" + +steps: + - input: "Input values for the variables" + key: "input-variables" + fields: + - select: "SERVERLESS_PROJECT" + key: "SERVERLESS_PROJECT" + options: + - label: "observability" + value: "observability" + - label: "security" + value: "security" + default: "observability" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":elastic: Serverless Integration tests" + key: test-serverless + command: ".buildkite/scripts/test_packages_with_serverless.sh" + env: + SERVERLESS_PROJECT: "${SERVERLESS_PROJECT:-observability}" + UPLOAD_SAFE_LOGS: 1 + artifact_paths: + - build/test-results/*.xml + - build/test-coverage/coverage-*.xml + + - wait: ~ + continue_on_failure: true + + - label: ":junit: Junit annotate" + plugins: + - junit-annotate#v2.4.1: + artifacts: "build/test-results/*.xml" + agents: + provider: "gcp" # junit plugin requires docker + +notify: + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 1960b6a69..51bbc9c9b 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -31,6 +31,22 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ ] + }, + { + "enabled": true, + "pipelineSlug": "elastic-package-test-serverless", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": false, + "build_on_commit": false, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:test)\\W+(?:serverless))$", + "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:test)\\W+(?:serverless))$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": [ ] } ] } diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index a8466db85..b8085cecf 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -5,10 +5,7 @@ source .buildkite/scripts/tooling.sh set -euo pipefail -WORKSPACE="$(pwd)" -TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package" - -cleanup() { +ensure_logout() { local error_code=$? if [ $error_code != 0 ] ; then @@ -18,14 +15,9 @@ cleanup() { fi fi - echo "Deleting temporal files..." - cd "${WORKSPACE}" - rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*" - echo "Done." - exit $error_code } -trap cleanup EXIT +trap ensure_logout EXIT usage() { echo "$0 [-t ] [-h]" @@ -39,15 +31,15 @@ PARALLEL_TARGET="test-check-packages-parallel" FALSE_POSITIVES_TARGET="test-check-packages-false-positives" KIND_TARGET="test-check-packages-with-kind" SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags" -TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json" ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"} REPO_NAME=$(repo_name "${BUILDKITE_REPO}") -REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)" +export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)" TARGET="" PACKAGE="" -while getopts ":t:p:h" o; do +SERVERLESS="false" +while getopts ":t:p:sh" o; do case "${o}" in t) TARGET=${OPTARG} @@ -55,6 +47,9 @@ while getopts ":t:p:h" o; do p) PACKAGE=${OPTARG} ;; + s) + SERVERLESS="true" + ;; h) usage exit 0 @@ -78,57 +73,37 @@ if [[ "${TARGET}" == "" ]]; then exit 1 fi -google_cloud_auth_safe_logs() { - local gsUtilLocation="" - gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}") - - local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME} - - echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}" - - google_cloud_auth "${secretFileLocation}" -} - -upload_safe_logs() { - local bucket="$1" - local source="$2" - local target="$3" - - if ! ls ${source} 2>&1 > /dev/null ; then - echo "upload_safe_logs: artifacts files not found, nothing will be archived" - return - fi - - google_cloud_auth_safe_logs - - gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}" - - google_cloud_logout_active_account -} - add_bin_path -echo "--- install go" -with_go +if [[ "$SERVERLESS" == "false" ]]; then + # If packages are tested with Serverless, these action are already performed + # here: .buildkite/scripts/test_packages_with_serverless.sh + echo "--- install go" + with_go -echo "--- install docker" -with_docker + echo "--- install docker" + with_docker -echo "--- install docker-compose plugin" -with_docker_compose_plugin + echo "--- install docker-compose plugin" + with_docker_compose_plugin +fi if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then echo "--- install kubectl & kind" with_kubernetes fi -echo "--- Run integration test ${TARGET}" +label="${TARGET}" +if [ -n "${PACKAGE}" ]; then + label="${label} - ${PACKAGE}" +fi +echo "--- Run integration test ${label}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then make install # allow to fail this command, to be able to upload safe logs set +e - make PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}" + make SERVERLESS="${SERVERLESS}" PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}" testReturnCode=$? set -e @@ -162,7 +137,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT fi if [ $testReturnCode != 0 ]; then - echo "make PACKAGE_UDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}" + echo "make SERVERLESS=${SERVERLESS} PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}" exit ${testReturnCode} fi diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh index af033dec1..a015b52f5 100755 --- a/.buildkite/scripts/release.sh +++ b/.buildkite/scripts/release.sh @@ -1,5 +1,8 @@ #!/bin/bash +source .buildkite/scripts/install_deps.sh +source .buildkite/scripts/tooling.sh + set -euo pipefail cleanup() { @@ -7,11 +10,9 @@ cleanup() { } trap cleanup exit -WORKSPACE="/tmp/bin-buildkite/" +export WORKSPACE="/tmp/bin-buildkite/" VERSION="" -source .buildkite/scripts/install_deps.sh -source .buildkite/scripts/tooling.sh add_bin_path with_go diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh index e1eadb53b..7da2e0856 100755 --- a/.buildkite/scripts/test-with-integrations.sh +++ b/.buildkite/scripts/test-with-integrations.sh @@ -3,20 +3,6 @@ source .buildkite/scripts/install_deps.sh set -euo pipefail -WORKSPACE="$(pwd)" - -TMP_FOLDER_TEMPLATE_BASE="tmp.${GITHUB_PR_BASE_REPO}" -TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" - -cleanup() { - echo "Deleting temporal files..." - cd "${WORKSPACE}" - rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*" - echo "Done." -} - -trap cleanup EXIT - add_bin_path echo "--- install gh cli" @@ -179,10 +165,10 @@ add_pr_comment() { local source_pr_number="$1" local integrations_pr_link="$2" - retry 3 \ - gh pr comment "${source_pr_number}" \ - --body "Created or updated PR in integrations repository to test this version. Check ${integrations_pr_link}" \ - --repo "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}" + add_github_comment \ + "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}" \ + "${source_pr_number}" \ + "Created or updated PR in integrations repository to test this version. Check ${integrations_pr_link}" } diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh new file mode 100644 index 000000000..6dde90745 --- /dev/null +++ b/.buildkite/scripts/test_packages_with_serverless.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/tooling.sh +source .buildkite/scripts/install_deps.sh + +set -euo pipefail + +UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"} + +SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt" +FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt" + +export SERVERLESS="true" +SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"} + +add_pr_comment() { + local source_pr_number="$1" + local buildkite_build="$2" + + add_github_comment \ + "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}" \ + "${source_pr_number}" \ + "Triggered serverless pipeline: ${buildkite_build}" +} + +echo "Running packages on Serverles project type: ${SERVERLESS_PROJECT}" +if running_on_buildkite; then + SERVERLESS_PROJECT="$(buildkite-agent meta-data get SERVERLESS_PROJECT --default ${SERVERLESS_PROJECT:-"observability"})" + buildkite-agent annotate "Serverless Project: ${SERVERLESS_PROJECT}" --context "ctx-info-${SERVERLESS_PROJECT}" --style "info" +fi + + +add_bin_path + +echo "--- install go" +with_go + +echo "--- Install docker" +with_docker +echo "--- Install docker-compose" +with_docker_compose_plugin + +if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + echo "--- install gh cli" + with_github_cli + + add_pr_comment "${BUILDKITE_PULL_REQUEST}" "${BUILDKITE_BUILD_URL}" +fi + +echo "--- Install elastic-package" +# Required to start the Serverless Elastic stack +make install + +prepare_serverless_stack + +echo "Waiting time to avoid getaddrinfo ENOTFOUND errors if any..." +sleep 120 +echo "Done." + +list_packages() { + find test/packages/parallel -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort +} + +any_package_failing=0 + +for package in $(list_packages); do + if ! .buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p "${package}" -s ; then + echo "- ${package}" >> "${FAILED_PACKAGES_FILE_PATH}" + any_package_failing=1 + fi +done + +if running_on_buildkite ; then + if [ -f "${SKIPPED_PACKAGES_FILE_PATH}" ]; then + create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" "${SKIPPED_PACKAGES_FILE_PATH}" "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}" + fi + + if [ -f "${FAILED_PACKAGES_FILE_PATH}" ]; then + create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" "${FAILED_PACKAGES_FILE_PATH}" "error" "ctx-failed-packages-${SERVERLESS_PROJECT}" + fi +fi + +if [ $any_package_failing -eq 1 ] ; then + echo "These packages have failed:" + cat "${FAILED_PACKAGES_FILE_PATH}" + exit 1 +fi diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh index 0c18d9502..3151774df 100755 --- a/.buildkite/scripts/tooling.sh +++ b/.buildkite/scripts/tooling.sh @@ -18,11 +18,17 @@ repo_name() { } buildkite_pr_branch_build_id() { - if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then + if [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then + echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}" + return + fi + + if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" ]]; then echo "${BUILDKITE_BRANCH}-${BUILDKITE_BUILD_NUMBER}" return fi - echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}" + # Other pipelines + echo "${BUILDKITE_BRANCH}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}" } google_cloud_auth() { @@ -74,3 +80,111 @@ running_on_buildkite() { fi return 1 } + +create_elastic_package_profile() { + local name="$1" + elastic-package profiles create "${name}" + elastic-package profiles use "${name}" +} + +prepare_serverless_stack() { + echo "--- Prepare serverless stack" + + # Creating a new profile allow to set a specific name for the serverless project + local profile_name="elastic-package-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}" + if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + profile_name="elastic-package-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}" + fi + create_elastic_package_profile "${profile_name}" + + export EC_API_KEY=${EC_API_KEY_SECRET} + export EC_HOST=${EC_HOST_SECRET} + + echo "Boot up the Elastic stack" + # Currently, if STACK_VERSION is not defined, for serverless it will be + # used as Elastic stack version (for agents) the default version in elastic-package + local stack_version=${STACK_VERSION:-""} + local args="-v" + if [ -n "${stack_version}" ]; then + args="${args} --version ${stack_version}" + fi + + # grep command required to remove password from the output + if ! elastic-package stack up \ + -d \ + ${args} \ + --provider serverless \ + -U "stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}" 2>&1 | grep -E -v "^Password: " ; then + return 1 + fi + echo "" + elastic-package stack status + echo "" +} + +google_cloud_auth_safe_logs() { + local gsUtilLocation="" + gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}") + + local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME} + + echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}" + + google_cloud_auth "${secretFileLocation}" +} + +upload_safe_logs() { + local bucket="$1" + local source="$2" + local target="$3" + + if ! ls ${source} 2>&1 > /dev/null ; then + echo "upload_safe_logs: artifacts files not found, nothing will be archived" + return + fi + + google_cloud_auth_safe_logs + + gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}" + + google_cloud_logout_active_account +} + +clean_safe_logs() { + rm -rf "${WORKSPACE}/build/elastic-stack-dump" + rm -rf "${WORKSPACE}/build/container-logs" +} + +cleanup() { + echo "Deleting temporary files..." + rm -rf ${WORKSPACE}/${TMP_FOLDER_TEMPLATE_BASE}.* + echo "Done." +} + +create_collapsed_annotation() { + local title="$1" + local file="$2" + local style="$3" + local context="$4" + + local annotation_file="tmp.annotation.md" + echo "
${title}" >> ${annotation_file} + echo -e "\n\n" >> ${annotation_file} + cat "${file}" >> ${annotation_file} + echo "
" >> ${annotation_file} + + cat ${annotation_file} | buildkite-agent annotate --style "${style}" --context "${context}" + + rm -f ${annotation_file} +} + +add_github_comment() { + local repository="$1" + local pr_id="$2" + local message="$3" + + retry 3 \ + gh pr comment "${pr_id}" \ + --body "${message}" \ + --repo "${repository}" +} diff --git a/README.md b/README.md index dca14997e..34b0944b2 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,26 @@ Usually, this process would require the following manual steps: - Creating this PR would automatically trigger a new Jenkins pipeline. +### Testing with Elastic serverless + +While working on a branch, it might be interesting to test your changes using +a project created in [Elastic serverless](https://docs.elastic.co/serverless), instead of spinning up a local +Elastic stack. To do so, you can add a new comment while developing in your Pull request +a comment like `test serverless`. + +Adding that comment in your Pull Request will create a new build of this +[Buildkite pipeline](https://buildkite.com/elastic/elastic-package-test-serverless). +This pipeline creates a new Serverless project and run some tests with the packages defined +in the `test/packages/parallel` folder. Currently, there are some differences with respect to testing +with a local Elastic stack: +- System tests are not executed. +- Disabled comparison of results in pipeline tests to avoid errors related to GeoIP fields +- Pipeline tests cannot be executed with coverage flags. + +At the same time, this pipeline is going to be triggered daily to test the latest contents +of the main branch with an Elastic serverless project. + + ## Commands `elastic-package` currently offers the commands listed below. diff --git a/catalog-info.yaml b/catalog-info.yaml index 4af584497..eca70afe5 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -155,3 +155,54 @@ spec: everyone: access_level: BUILD_AND_READ +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elastic-package-test-serverless + description: Pipeline to test elastic-package with Serverless projects + links: + - title: Pipeline + url: https://buildkite.com/elastic/elastic-package-test-serverless + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: elastic-package-test-serverless + description: Pipeline to test elastic-package with Serverless projects + spec: + pipeline_file: ".buildkite/pipeline.serverless.yml" + provider_settings: + build_pull_request_forks: false + # Managed by buildkite-pr-bot + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + publish_commit_status: false # do not update status of commits for this pipeline + build_tags: false + build_branches: false + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api') + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: '!main' + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: '!main' + repository: elastic/elastic-package + schedules: + Test Serverless Daily: + branch: main + cronline: "00 5 * * *" + message: Test Serverless Daily + env: + SERVERLESS_PROJECT: observability + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ + diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 5628c7a42..40422e7ba 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -20,8 +20,14 @@ cleanup() { kind delete cluster || true fi - # Take down the stack - elastic-package stack down -v + # In case it is tested with Elatic serverless, there should be just one Elastic stack + # started to test all packages. In our CI, this Elastic serverless stack is started + # at the beginning of the pipeline and must be running for all packages without stopping it between + # packages. + if [[ "$SERVERLESS" != "true" ]]; then + # Take down the stack + elastic-package stack down -v + fi if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then # Delete the logstash profile @@ -45,6 +51,7 @@ ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDE export ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" export ELASTIC_PACKAGE_LINKS_FILE_PATH +export SERVERLESS=${SERVERLESS:-"false"} OLDPWD=$PWD # Build/check packages @@ -68,13 +75,18 @@ if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then echo "stack.logstash_enabled: true" >> ~/.elastic-package/profiles/logstash/config.yml fi -# Update the stack -elastic-package stack update -v +# In case it is tested with Elatic serverless, there should be just one Elastic stack +# started to test all packages. In our CI, this Elastic serverless stack is started +# at the beginning of the pipeline and must be running for all packages. +if [[ "${SERVERLESS}" != "true" ]]; then + # Update the stack + elastic-package stack update -v -# Boot up the stack -elastic-package stack up -d -v + # Boot up the stack + elastic-package stack up -d -v -elastic-package stack status + elastic-package stack status +fi if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Boot up the kind cluster @@ -113,13 +125,19 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do echo "Package \"${package_to_test}\" skipped: not supported with Elastic Agent running in the stack (missing capabilities)." exit # as it is run in a subshell, it cannot be used "continue" fi + + if [[ "${SERVERLESS}" == "true" ]]; then + # skip system tests + elastic-package test asset -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic + elastic-package test static -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic + # FIXME: adding test-coverage for serverless results in errors like this: + # Error: error running package pipeline tests: could not complete test run: error calculating pipeline coverage: error fetching pipeline stats for code coverage calculations: need exactly one ES node in stats response (got 4) + elastic-package test pipeline -v --report-format xUnit --report-output file --defer-cleanup 1s + exit # as it is run in a subshell, it cannot be used "continue" + fi + # Run all tests # defer-cleanup is set to a short period to verify that the option is available - elastic-package test -v \ - --report-format xUnit \ - --report-output file \ - --defer-cleanup 1s \ - --test-coverage \ - --coverage-format=generic + elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic fi ) cd - diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index 3f5b817a7..bb3b80558 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -114,6 +114,26 @@ Usually, this process would require the following manual steps: - Creating this PR would automatically trigger a new Jenkins pipeline. +### Testing with Elastic serverless + +While working on a branch, it might be interesting to test your changes using +a project created in [Elastic serverless](https://docs.elastic.co/serverless), instead of spinning up a local +Elastic stack. To do so, you can add a new comment while developing in your Pull request +a comment like `test serverless`. + +Adding that comment in your Pull Request will create a new build of this +[Buildkite pipeline](https://buildkite.com/elastic/elastic-package-test-serverless). +This pipeline creates a new Serverless project and run some tests with the packages defined +in the `test/packages/parallel` folder. Currently, there are some differences with respect to testing +with a local Elastic stack: +- System tests are not executed. +- Disabled comparison of results in pipeline tests to avoid errors related to GeoIP fields +- Pipeline tests cannot be executed with coverage flags. + +At the same time, this pipeline is going to be triggered daily to test the latest contents +of the main branch with an Elastic serverless project. + + ## Commands `elastic-package` currently offers the commands listed below.