From f79ac77ad0d33b46441e4c797e191130fc9d96b7 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 23 Sep 2021 21:52:55 +0200 Subject: [PATCH 01/55] Update to v7.15.0 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index f1db7417e1..9cadc655af 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.14.1 +ELK_VERSION=7.15.0 diff --git a/README.md b/README.md index 337f050734..d1bbd53021 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.14.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -256,7 +256,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.14.1' \ + -H 'kbn-version: 7.15.0' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index 8d7d7db19a..105c9a5321 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1 + image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.14.1 + image: docker.elastic.co/logstash/logstash:7.15.0 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.14.1 + image: docker.elastic.co/kibana/kibana:7.15.0 ports: - "5601:5601" configs: From 47fb64f9c680e6c19344f2279b08587d04ab5c91 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 23 Sep 2021 22:02:48 +0200 Subject: [PATCH 02/55] Update link to security reference The tutorial page was deleted and links to this reference page. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d1bbd53021..4ab9fa942b 100644 --- a/README.md +++ b/README.md @@ -207,8 +207,8 @@ users][builtin-users] instead for increased security. $ docker-compose restart kibana logstash ``` - *:information_source: Learn more about the security of the Elastic stack at [Tutorial: Getting started with - security][sec-tutorial].* + *:information_source: Learn more about the security of the Elastic stack at [Secure the Elastic + Stack][sec-cluster].* ### Injecting data @@ -435,7 +435,7 @@ instead of `elasticsearch`.* [builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html [ls-security]: https://www.elastic.co/guide/en/logstash/current/ls-security.html -[sec-tutorial]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-getting-started.html +[sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-cluster.html [connect-kibana]: https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html [index-pattern]: https://www.elastic.co/guide/en/kibana/current/index-patterns.html From 791150114ae4a393ea937b0555128ebbe8f68e1c Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 23 Sep 2021 23:00:17 +0200 Subject: [PATCH 03/55] Update instructions for Docker Desktop --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4ab9fa942b..6817383851 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p 1. [Requirements](#requirements) * [Host setup](#host-setup) * [SELinux](#selinux) - * [Docker for Desktop](#docker-for-desktop) + * [Docker Desktop](#docker-desktop) * [Windows](#windows) * [macOS](#macos) 1. [Usage](#usage) @@ -105,17 +105,18 @@ apply the proper context: $ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ ``` -### Docker for Desktop +### Docker Desktop #### Windows -Ensure the [Shared Drives][win-shareddrives] feature is enabled for the `C:` drive. +If you are using the legacy Hyper-V mode of _Docker Desktop for Windows_, ensure [File Sharing][win-filesharing] is +enabled for the `C:` drive. #### macOS -The default Docker for Mac configuration allows mounting files from `/Users/`, `/Volumes/`, `/private/`, and `/tmp` -exclusively. Make sure the repository is cloned in one of those locations or follow the instructions from the -[documentation][mac-mounts] to add more locations. +The default configuration of _Docker Desktop for Mac_ allows mounting files from `/Users/`, `/Volume/`, `/private/`, +`/tmp` and `/var/folders` exclusively. Make sure the repository is cloned in one of those locations or follow the +instructions from the [documentation][mac-filesharing] to add more locations. ## Usage @@ -430,8 +431,8 @@ instead of `elasticsearch`.* [booststap-checks]: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html [es-sys-config]: https://www.elastic.co/guide/en/elasticsearch/reference/current/system-config.html -[win-shareddrives]: https://docs.docker.com/docker-for-windows/#shared-drives -[mac-mounts]: https://docs.docker.com/docker-for-mac/osxfs/ +[win-filesharing]: https://docs.docker.com/desktop/windows/#file-sharing +[mac-filesharing]: https://docs.docker.com/desktop/mac/#file-sharing [builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html [ls-security]: https://www.elastic.co/guide/en/logstash/current/ls-security.html From b3abdb1d2e48659d2c8c99af8a5fc74165c5593b Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Wed, 23 Jun 2021 20:37:20 +0200 Subject: [PATCH 04/55] ci: Use Compose v2 --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf8a326ecf..3caaf69e11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: ##################################################### - name: Prepare environment + env: + COMPOSE_VERSION: v2.0.1 run: | # Install Linux packages @@ -33,9 +35,18 @@ jobs: sudo apt install -y expect + # Enable support for Compose V2 + # + # Instructions: + # https://docs.docker.com/compose/cli-command/#install-on-linux + + mkdir -p ~/.docker/cli-plugins + curl -sL "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose + chmod +x ~/.docker/cli-plugins/docker-compose + # Pre-build container images - docker-compose build + docker compose build ######################################################## # # @@ -57,7 +68,7 @@ jobs: # Run Elasticsearch and wait for its availability - docker-compose up -d elasticsearch + docker compose up -d elasticsearch source .github/workflows/scripts/lib/testing.sh poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme' @@ -72,7 +83,7 @@ jobs: ########################################################## - name: Run the stack - run: docker-compose up -d + run: docker compose up -d - name: Execute core test suite run: .github/workflows/scripts/run-tests-core.sh @@ -82,10 +93,10 @@ jobs: # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#job-status-check-functions if: always() run: | - docker-compose ps - docker-compose logs elasticsearch - docker-compose logs logstash - docker-compose logs kibana + docker compose ps + docker compose logs elasticsearch + docker compose logs logstash + docker compose logs kibana ############################## # # @@ -104,26 +115,29 @@ jobs: sed -i '$ a input { udp { port => 5000 codec => json } }' logstash/pipeline/logstash.conf + # Restart Logstash for changes to take effect + + docker compose restart logstash + # Run Logspout and execute tests. - # Logstash will be restarted as a result of building the Logspout - # image, so changes above will automatically take effect. - docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up -d logspout + docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up -d logspout .github/workflows/scripts/run-tests-logspout.sh # Revert changes to Logstash configuration sed -i '/input { udp { port => 5000 codec => json } }/d' logstash/pipeline/logstash.conf + docker compose restart logstash - name: 'debug: Display state and logs (Logspout)' if: always() run: | - docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps - docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout + docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps + docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout # next steps don't need Logspout - docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml stop logspout + docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml stop logspout # next steps don't need Logstash - docker-compose stop logstash + docker compose stop logstash # # Enterprise Search @@ -138,25 +152,25 @@ jobs: # Restart Elasticsearch for changes to take effect - docker-compose restart elasticsearch + docker compose restart elasticsearch # Run Enterprise Search and execute tests - docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -d enterprise-search + docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -d enterprise-search .github/workflows/scripts/run-tests-enterprise-search.sh # Revert changes to Elasticsearch configuration sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml - docker-compose restart elasticsearch + docker compose restart elasticsearch - name: 'debug: Display state and logs (Enterprise Search)' if: always() run: | - docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml ps - docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search + docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml ps + docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search # next steps don't need Enterprise Search - docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml stop enterprise-search + docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml stop enterprise-search # # APM Server @@ -164,16 +178,16 @@ jobs: - name: Execute APM Server test suite run: | - docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up -d apm-server + docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up -d apm-server .github/workflows/scripts/run-tests-apm-server.sh - name: 'debug: Display state and logs (APM Server)' if: always() run: | - docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml ps - docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml logs apm-server + docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml ps + docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml logs apm-server # next steps don't need APM Server - docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml stop apm-server + docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml stop apm-server # # Metricbeat @@ -181,16 +195,16 @@ jobs: - name: Execute Metricbeat test suite run: | - docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up -d metricbeat + docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up -d metricbeat .github/workflows/scripts/run-tests-metricbeat.sh - name: 'debug: Display state and logs (Metricbeat)' if: always() run: | - docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps - docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat + docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps + docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat # next steps don't need Metricbeat - docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat + docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat ############## # # @@ -201,7 +215,7 @@ jobs: - name: Terminate all components if: always() run: >- - docker-compose + docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml From 9a541d14f89d006289e9b15fa58c09204df54754 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 7 Oct 2021 01:10:53 +0200 Subject: [PATCH 05/55] ci: Fix Metricbeat test The naming scheme of containers created by Compose has changed in V2. Ref. docker/compose#8655 --- .github/workflows/scripts/run-tests-metricbeat.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/run-tests-metricbeat.sh b/.github/workflows/scripts/run-tests-metricbeat.sh index b9909eb820..46d0d7d554 100755 --- a/.github/workflows/scripts/run-tests-metricbeat.sh +++ b/.github/workflows/scripts/run-tests-metricbeat.sh @@ -25,7 +25,7 @@ poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" # agent.type:"metricbeat" # AND event.module:"docker" # AND event.dataset:"docker.container" -# AND container.name:"docker-elk_elasticsearch_1" +# AND container.name:"docker-elk-elasticsearch-1" # log 'Searching a document generated by Metricbeat' @@ -34,7 +34,7 @@ declare -i count # retry for max 60s (30*2s) for _ in $(seq 1 30); do - response="$(curl "http://${ip_es}:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk_elasticsearch_1%22&pretty" -s -u elastic:testpasswd)" + response="$(curl "http://${ip_es}:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)" set +u # prevent "unbound variable" if assigned value is not an integer count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" From 51b8525767780f75d8267e51c5545b0f919ac492 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 7 Oct 2021 01:27:27 +0200 Subject: [PATCH 06/55] ci: Build/pull all images ahead of actual tests --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3caaf69e11..c12462b30d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,13 @@ jobs: # Pre-build container images - docker compose build + docker compose \ + -f docker-compose.yml \ + -f extensions/logspout/logspout-compose.yml \ + -f extensions/enterprise-search/enterprise-search-compose.yml \ + -f extensions/apm-server/apm-server-compose.yml \ + -f extensions/metricbeat/metricbeat-compose.yml \ + build ######################################################## # # From 04fdb981ebc3b4964d2de67dcb4ce7992d3406bd Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Tue, 26 Oct 2021 16:01:22 +0200 Subject: [PATCH 07/55] Restore "Send automated PR upon new stack release" This reverts commit 9016e00e7bf170f31d025eff838cfb5209bbdd57 and updates the 'latest' version detection logic to iterate over releases and pick the latest matching semver, instead of relying on GitHub's '/releases/latest' API. --- .github/workflows/update.yml | 78 ++++++++++++++++++++++++++++++++++++ .gitignore | 3 ++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/update.yml create mode 100644 .gitignore diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml new file mode 100644 index 0000000000..2fb4f264e3 --- /dev/null +++ b/.github/workflows/update.yml @@ -0,0 +1,78 @@ +name: Update Elastic release + +on: + schedule: + - cron: '0 0 * * 0' # At 00:00 every Sunday + +jobs: + + check-and-update: + name: Check and update Elastic release + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + - run: npm install semver + + - name: Get latest release version + uses: actions/github-script@v5 + id: get-latest-release + with: + script: | + const semver = require('semver') + + const latestVersion = await github. + paginate(github.rest.repos.listReleases, { + owner: 'elastic', + repo: 'elasticsearch' + }) + .then(releases => { + for (const release of releases) { + // Results are returned sorted by created_at, so it is safe to assume + // that the first encountered match is also the series' latest release. + + const version=semver.clean(release.tag_name) + + if (semver.satisfies(version, '7.x')) { + return version + } + } + }); + + if (latestVersion) { + // Return an object so that the result can be handled as structured data + // instead of a quoted string in subsequent steps. + return { version: latestVersion } + } + + - name: Update stack version + id: update-files + if: steps.get-latest-release.outputs.result + run: | + source .env + cur_ver="$ELK_VERSION" + new_ver=${{ fromJson(steps.get-latest-release.outputs.result).version }} + + # Escape dot characters so sed interprets them as literal dots + cur_ver="$(echo $cur_ver | sed 's/\./\\./g')" + + for f in .env docker-stack.yml README.md; do + sed -i "s/${cur_ver}/${new_ver}/g" "$f" + done + + git_status="$(git status --porcelain)" + if [[ ${git_status} ]]; then + echo -e 'Changes to be committed:\n' + echo "${git_status}" + echo '::set-output name=has_changes::true' + fi + + - name: Send pull request to update to new version + if: steps.update-files.outputs.has_changes + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} + title: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} + delete-branch: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..753765ea2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# Node.js artifacts created by certain GitHub Actions +/node_modules/ +/package-lock.json From b2e2aa792c92524940a7c4af97382c2a8b2d4cf3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 19:41:15 +0200 Subject: [PATCH 08/55] Update to v7.15.1 (#631) Co-authored-by: antoineco --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 9cadc655af..4ec13bdfed 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.15.0 +ELK_VERSION=7.15.1 diff --git a/README.md b/README.md index 6817383851..2cc3ab1891 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -257,7 +257,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.15.0' \ + -H 'kbn-version: 7.15.1' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index 105c9a5321..9a40ee3f37 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.15.0 + image: docker.elastic.co/logstash/logstash:7.15.1 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.15.0 + image: docker.elastic.co/kibana/kibana:7.15.1 ports: - "5601:5601" configs: From c8f5f01b7f05b43f47dc23a86994ba7368ce60b1 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Tue, 26 Oct 2021 22:12:20 +0200 Subject: [PATCH 09/55] ci: Update all release series from single workflow Run the update job for each currently supported release series, so that automated updates can be orchestrated from the main branch via a single scheduled workflow. --- .github/workflows/update.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 2fb4f264e3..c2725e9497 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -9,9 +9,21 @@ jobs: check-and-update: name: Check and update Elastic release runs-on: ubuntu-latest + strategy: + matrix: + release: + - 7.x + - 6.x + include: + - release: 7.x + branch: main + - release: 6.x + branch: release-6.x steps: - uses: actions/checkout@v2 + with: + ref: ${{ matrix.branch }} - uses: actions/setup-node@v2 - run: npm install semver @@ -35,7 +47,7 @@ jobs: const version=semver.clean(release.tag_name) - if (semver.satisfies(version, '7.x')) { + if (semver.satisfies(version, '${{ matrix.release }}')) { return version } } @@ -73,6 +85,7 @@ jobs: if: steps.update-files.outputs.has_changes uses: peter-evans/create-pull-request@v3 with: + branch: update/${{ matrix.branch }} commit-message: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} title: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} delete-branch: true From 29d105e663fb84afb7ce817504a6672de6c292f1 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Tue, 26 Oct 2021 23:42:44 +0200 Subject: [PATCH 10/55] ci(update): Checkout repo after running Node.js steps Ensures the workspace is clean before attempting to update any file. Removes the need for explicitly ignoring Node.js artifacts in Git, since they get cleaned up by the 'checkout' action before fetching. --- .github/workflows/update.yml | 9 +++++---- .gitignore | 3 --- 2 files changed, 5 insertions(+), 7 deletions(-) delete mode 100644 .gitignore diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index c2725e9497..ed342524c5 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -21,10 +21,6 @@ jobs: branch: release-6.x steps: - - uses: actions/checkout@v2 - with: - ref: ${{ matrix.branch }} - - uses: actions/setup-node@v2 - run: npm install semver @@ -59,6 +55,11 @@ jobs: return { version: latestVersion } } + - uses: actions/checkout@v2 + if: steps.get-latest-release.outputs.result + with: + ref: ${{ matrix.branch }} + - name: Update stack version id: update-files if: steps.get-latest-release.outputs.result diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 753765ea2b..0000000000 --- a/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Node.js artifacts created by certain GitHub Actions -/node_modules/ -/package-lock.json From c3cc7ae15c47c75e36bec757d981cf47fdb8a990 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 28 Oct 2021 15:05:17 +0200 Subject: [PATCH 11/55] ci: Check for readiness of Logstash in Logspout test Since we restart Logstash to enable the UDP input prior to running this test, we should also ensure it has successfully come back, otherwise we start searching for the expected documents in Elasticsearch way too early and often run into timeouts. --- .github/workflows/scripts/run-tests-logspout.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh index ca2d327550..cebd4d8c9f 100755 --- a/.github/workflows/scripts/run-tests-logspout.sh +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -8,16 +8,21 @@ source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" cid_es="$(container_id elasticsearch)" -cid_ls="$(container_id logspout)" +cid_ls="$(container_id logstash)" +cid_lsp="$(container_id logspout)" ip_es="$(service_ip elasticsearch)" -ip_ls="$(service_ip logspout)" +ip_ls="$(service_ip logstash)" +ip_lsp="$(service_ip logspout)" log 'Waiting for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +log 'Waiting for readiness of Logstash' +poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty" + log 'Waiting for readiness of Logspout' -poll_ready "$cid_ls" "http://${ip_ls}/health" +poll_ready "$cid_lsp" "http://${ip_lsp}/health" # When Logspout starts, it prints the following log line: # 2021/01/07 16:14:52 # logspout v3.2.13-custom by gliderlabs From 9fde1ac3512ea01fd3f73c50b78e9c3d7af756f0 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 28 Oct 2021 16:41:49 +0200 Subject: [PATCH 12/55] ci(core): Retry sending data over TCP in case of error The first attempt occasionally fails, probably due to a race with the listener's startup. Additionally, perform minor adjustments to the output written to stderr to ensure it is properly flushed in the CI environment. --- .github/workflows/scripts/lib/testing.sh | 14 +++++++++++ .github/workflows/scripts/run-tests-core.sh | 24 +++++++++++++++---- .../workflows/scripts/run-tests-logspout.sh | 12 +++++++--- .../workflows/scripts/run-tests-metricbeat.sh | 18 ++++++++++---- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scripts/lib/testing.sh b/.github/workflows/scripts/lib/testing.sh index 66a6d5b474..1099d1eef0 100755 --- a/.github/workflows/scripts/lib/testing.sh +++ b/.github/workflows/scripts/lib/testing.sh @@ -23,6 +23,8 @@ function container_id { local cid + local -i was_retried=0 + # retry for max 60s (30*2s) for _ in $(seq 1 30); do cid="$(docker container ls -aq -f label="$label")" @@ -30,9 +32,14 @@ function container_id { break fi + was_retried=1 echo -n '.' >&2 sleep 2 done + if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 + fi if [ -z "${cid:-}" ]; then err "Timed out waiting for creation of container with label ${label}" @@ -91,6 +98,8 @@ function poll_ready { local -i result=1 local output + local -i was_retried=0 + # retry for max 300s (60*5s) for _ in $(seq 1 60); do if [[ $(docker container inspect "$cid" --format '{{ .State.Status}}') == 'exited' ]]; then @@ -104,9 +113,14 @@ function poll_ready { break fi + was_retried=1 echo -n 'x' >&2 sleep 5 done + if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 + fi echo -e "\n${output::-3}" diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index 0ca8498953..5757d162ee 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -37,15 +37,31 @@ log 'Searching index pattern via Kibana API' response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" echo "$response" count="$(jq -rn --argjson data "${response}" '$data.total')" -if [[ $count -ne 1 ]]; then +if (( count != 1 )); then echo "Expected 1 index pattern, got ${count}" exit 1 fi log 'Sending message to Logstash TCP input' -echo 'dockerelk' | nc -q0 "$ip_ls" 5000 -sleep 1 +declare -i was_retried=0 + +# retry for max 10s (5*2s) +for _ in $(seq 1 5); do + if echo 'dockerelk' | nc -q0 "$ip_ls" 5000; then + break + fi + + was_retried=1 + echo -n 'x' >&2 + sleep 2 +done +if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 +fi + +sleep 3 curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \ -s -w '\n' @@ -53,7 +69,7 @@ log 'Searching message in Elasticsearch' response="$(curl "http://${ip_es}:9200/logstash-*/_count?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" echo "$response" count="$(jq -rn --argjson data "${response}" '$data.count')" -if [[ $count -ne 1 ]]; then +if (( count != 1 )); then echo "Expected 1 document, got ${count}" exit 1 fi diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh index cebd4d8c9f..caf62bcbe3 100755 --- a/.github/workflows/scripts/run-tests-logspout.sh +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -35,6 +35,8 @@ log 'Searching a log entry forwarded by Logspout' declare response declare -i count +declare -i was_retried=0 + # retry for max 60s (30*2s) for _ in $(seq 1 30); do response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" @@ -47,15 +49,19 @@ for _ in $(seq 1 30); do break fi + was_retried=1 echo -n 'x' >&2 sleep 2 done -echo -e '\n' >&2 +if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 +fi echo "$response" # Logspout may restart if Logstash isn't ready yet, so we tolerate multiple # results -if (( count < 1 )); then - echo "Expected at least 1 document, got ${count}" +if (( count == 0 )); then + echo 'Expected at least 1 document' exit 1 fi diff --git a/.github/workflows/scripts/run-tests-metricbeat.sh b/.github/workflows/scripts/run-tests-metricbeat.sh index 46d0d7d554..b0e9856223 100755 --- a/.github/workflows/scripts/run-tests-metricbeat.sh +++ b/.github/workflows/scripts/run-tests-metricbeat.sh @@ -19,8 +19,8 @@ poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' log 'Waiting for readiness of Metricbeat' poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" -# We expect to find one monitoring entry for the 'elasticsearch' Compose -# service using the following query: +# We expect to find monitoring entries for the 'elasticsearch' Compose service +# using the following query: # # agent.type:"metricbeat" # AND event.module:"docker" @@ -32,6 +32,8 @@ log 'Searching a document generated by Metricbeat' declare response declare -i count +declare -i was_retried=0 + # retry for max 60s (30*2s) for _ in $(seq 1 30); do response="$(curl "http://${ip_es}:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)" @@ -44,13 +46,19 @@ for _ in $(seq 1 30); do break fi + was_retried=1 echo -n 'x' >&2 sleep 2 done -echo -e '\n' >&2 +if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 +fi echo "$response" -if (( count != 1 )); then - echo "Expected 1 document, got ${count}" +# Metricbeat buffers metrics until Elasticsearch becomes ready, so we tolerate +# multiple results +if (( count == 0 )); then + echo 'Expected at least 1 document' exit 1 fi From 5ffa296608ec5a6fe77b75a9ba893f734ee2cc03 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 30 Oct 2021 18:55:06 +0200 Subject: [PATCH 13/55] doc: Add note about cancelling an ongoing trial --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cc3ab1891..24140a928a 100644 --- a/README.md +++ b/README.md @@ -308,9 +308,12 @@ containers: [Configuring Logstash for Docker][ls-docker]. ### How to disable paid features -Switch the value of Elasticsearch's `xpack.license.self_generated.type` option from `trial` to `basic` (see [License +Switch the value of Elasticsearch's `xpack.license.self_generated.type` setting from `trial` to `basic` (see [License settings][trial-license]). +You can also cancel an ongoing trial before its expiry date — and thus revert to a basic license — either from the +[License Management][license-mngmt] panel of Kibana, or using Elasticsearch's [Licensing APIs][license-apis]. + ### How to scale out the Elasticsearch cluster Follow the instructions from the Wiki: [Scaling out Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) @@ -423,6 +426,8 @@ instead of `elasticsearch`.* [xpack]: https://www.elastic.co/what-is/open-x-pack [paid-features]: https://www.elastic.co/subscriptions [trial-license]: https://www.elastic.co/guide/en/elasticsearch/reference/current/license-settings.html +[license-mngmt]: https://www.elastic.co/guide/en/kibana/current/managing-licenses.html +[license-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html [elastdocker]: https://github.com/sherifabdlnaby/elastdocker From 233c4e0b8cb816bac7807846c1fa7601f17b27ed Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Tue, 2 Nov 2021 10:45:37 +0100 Subject: [PATCH 14/55] ci: Use token from GitHub App in auto-update workflow --- .github/workflows/update.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index ed342524c5..a22cfc6e87 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -82,10 +82,19 @@ jobs: echo '::set-output name=has_changes::true' fi + - name: Impersonate update bot + uses: tibdex/github-app-token@v1 + id: generate-token + if: steps.update-files.outputs.has_changes + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Send pull request to update to new version if: steps.update-files.outputs.has_changes uses: peter-evans/create-pull-request@v3 with: + token: ${{ steps.generate-token.outputs.token }} branch: update/${{ matrix.branch }} commit-message: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} title: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} From e9903fecf6186444dc354dac1cb56035e6cb7741 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 6 Nov 2021 10:38:41 +0100 Subject: [PATCH 15/55] ci: Run tests using latest Compose V2 release --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c12462b30d..a558020eb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,8 +24,6 @@ jobs: ##################################################### - name: Prepare environment - env: - COMPOSE_VERSION: v2.0.1 run: | # Install Linux packages @@ -41,8 +39,9 @@ jobs: # https://docs.docker.com/compose/cli-command/#install-on-linux mkdir -p ~/.docker/cli-plugins - curl -sL "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose + curl -sL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose + docker compose version # Pre-build container images From b826d7bbdff93243f2852f7f5c28d5f136c8e175 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 6 Nov 2021 12:06:31 +0100 Subject: [PATCH 16/55] ci: Always use _search API instead of _count Makes it easier to troubleshoot returned documents by inspecting the logs of CI runs. --- .github/workflows/scripts/run-tests-core.sh | 5 +++-- .github/workflows/scripts/run-tests-enterprise-search.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index 5757d162ee..45a25052a0 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -36,6 +36,7 @@ curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \ log 'Searching index pattern via Kibana API' response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" echo "$response" +declare -i count count="$(jq -rn --argjson data "${response}" '$data.total')" if (( count != 1 )); then echo "Expected 1 index pattern, got ${count}" @@ -66,9 +67,9 @@ curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \ -s -w '\n' log 'Searching message in Elasticsearch' -response="$(curl "http://${ip_es}:9200/logstash-*/_count?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" +response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" echo "$response" -count="$(jq -rn --argjson data "${response}" '$data.count')" +count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" if (( count != 1 )); then echo "Expected 1 document, got ${count}" exit 1 diff --git a/.github/workflows/scripts/run-tests-enterprise-search.sh b/.github/workflows/scripts/run-tests-enterprise-search.sh index fb1ab99a6e..3a58afd500 100755 --- a/.github/workflows/scripts/run-tests-enterprise-search.sh +++ b/.github/workflows/scripts/run-tests-enterprise-search.sh @@ -20,10 +20,10 @@ log 'Waiting for readiness of Enterprise Search' poll_ready "$cid_en" "http://${ip_en}:3002/api/ent/v1/internal/health" -u 'elastic:testpasswd' log 'Ensuring that App Search API keys were created in Elasticsearch' -response="$(curl "http://${ip_es}:9200/.ent-search-actastic-app_search_api_tokens_v3/_count?pretty" -s -u elastic:testpasswd)" +response="$(curl "http://${ip_es}:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?q=*:*&pretty" -s -u elastic:testpasswd)" echo "$response" declare -i count -count="$(jq -rn --argjson data "${response}" '$data.count')" +count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" if (( count != 2)); then echo "Expected search and private keys, got ${count} result(s)" exit 1 From e8d16db9b654f7d5f3ec6d1f58e5e814094a30c7 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 6 Nov 2021 20:33:21 +0100 Subject: [PATCH 17/55] doc: Add animated demo to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 24140a928a..ae6a3b2d75 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker It gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and the visualization power of Kibana. +![Animated demo](https://user-images.githubusercontent.com/3299086/140621647-2c06e2e6-81fc-4052-96b8-626b9d5f011f.gif) + *:information_source: The Docker images backing this stack include [X-Pack][xpack] with [paid features][paid-features] enabled by default (see [How to disable paid features](#how-to-disable-paid-features) to disable them). **The [trial license][trial-license] is valid for 30 days**. After this license expires, you can continue using the free features From 6f8117e2d82f2e847b7f50c532d8de13976cc304 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sun, 7 Nov 2021 11:46:55 +0100 Subject: [PATCH 18/55] doc: Increase resolution of animated demo, compress GIF --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae6a3b2d75..200287273d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker It gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and the visualization power of Kibana. -![Animated demo](https://user-images.githubusercontent.com/3299086/140621647-2c06e2e6-81fc-4052-96b8-626b9d5f011f.gif) +![Animated demo](https://user-images.githubusercontent.com/3299086/140641708-cea70d17-cc04-459f-89d9-3fcb5c58bc35.gif) *:information_source: The Docker images backing this stack include [X-Pack][xpack] with [paid features][paid-features] enabled by default (see [How to disable paid features](#how-to-disable-paid-features) to disable them). **The [trial From 9a523626eff3da32492895244051a8cea75a44de Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Tue, 9 Nov 2021 01:36:45 +0100 Subject: [PATCH 19/55] ci: Automatically merge bot PRs for Elastic updates Merges pull requests from docker-elk-updater[bot] if CI checks pass. --- .github/workflows/update-merge.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/update-merge.yml diff --git a/.github/workflows/update-merge.yml b/.github/workflows/update-merge.yml new file mode 100644 index 0000000000..f88fe7ffbc --- /dev/null +++ b/.github/workflows/update-merge.yml @@ -0,0 +1,24 @@ +name: Merge Elastic updates + +on: + workflow_run: + workflows: [ CI ] + types: + - completed + branches: + - update/main + - update/release-6.x + +jobs: + + merge: + name: Merge pull request + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + + steps: + - name: Approve and merge + uses: ridedott/merge-me-action@v2 + with: + GITHUB_LOGIN: docker-elk-updater + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8055143d52d7d50522c4953db7cd2986bbba614f Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 14:36:47 +0000 Subject: [PATCH 20/55] Update to v7.15.2 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 4ec13bdfed..753d5828ac 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.15.1 +ELK_VERSION=7.15.2 diff --git a/README.md b/README.md index 200287273d..1954db91e7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -259,7 +259,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.15.1' \ + -H 'kbn-version: 7.15.2' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index 9a40ee3f37..19919b42eb 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 + image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.15.1 + image: docker.elastic.co/logstash/logstash:7.15.2 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.15.1 + image: docker.elastic.co/kibana/kibana:7.15.2 ports: - "5601:5601" configs: From e6f9ec49d23572d3d8cfdbb5e1b5459e4ef86330 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 11 Nov 2021 19:08:42 +0100 Subject: [PATCH 21/55] ci: Delete branch after successful Elastic update merge --- .github/workflows/update-merge.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/update-merge.yml b/.github/workflows/update-merge.yml index f88fe7ffbc..b5fbf741d0 100644 --- a/.github/workflows/update-merge.yml +++ b/.github/workflows/update-merge.yml @@ -22,3 +22,13 @@ jobs: with: GITHUB_LOGIN: docker-elk-updater GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Delete branch + uses: actions/github-script@v5 + with: + script: | + await github.request('DELETE /repos/{owner}/{repo}/git/refs/{ref}', { + owner: '${{ github.event.workflow_run.repository.owner.login }}', + repo: '${{ github.event.workflow_run.repository.name }}', + ref: 'heads/${{ github.event.workflow_run.head_branch }}' + }) From 01a0cf94d54398b33addef5e8d109dcd09646427 Mon Sep 17 00:00:00 2001 From: Yuri Pereira Constante Date: Mon, 29 Nov 2021 05:28:17 -0300 Subject: [PATCH 22/55] Podman and SELinux compatible volume declaration (#639) --- README.md | 11 ----------- docker-compose.yml | 24 +++++------------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 1954db91e7..dd43be9076 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p 1. [Requirements](#requirements) * [Host setup](#host-setup) - * [SELinux](#selinux) * [Docker Desktop](#docker-desktop) * [Windows](#windows) * [macOS](#macos) @@ -97,16 +96,6 @@ By default, the stack exposes the following ports: Elastic stack in development environments. For production setups, we recommend users to set up their host according to the instructions from the Elasticsearch documentation: [Important System Configuration][es-sys-config].** -### SELinux - -On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux -into Permissive mode in order for docker-elk to start properly. For example on Redhat and CentOS, the following will -apply the proper context: - -```console -$ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ -``` - ### Docker Desktop #### Windows diff --git a/docker-compose.yml b/docker-compose.yml index 669e3373c5..c7429e5c83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,13 +7,8 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - type: bind - source: ./elasticsearch/config/elasticsearch.yml - target: /usr/share/elasticsearch/config/elasticsearch.yml - read_only: true - - type: volume - source: elasticsearch - target: /usr/share/elasticsearch/data + - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z + - elasticsearch:/usr/share/elasticsearch/data:z ports: - "9200:9200" - "9300:9300" @@ -32,14 +27,8 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - type: bind - source: ./logstash/config/logstash.yml - target: /usr/share/logstash/config/logstash.yml - read_only: true - - type: bind - source: ./logstash/pipeline - target: /usr/share/logstash/pipeline - read_only: true + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,z + - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,z ports: - "5044:5044" - "5000:5000/tcp" @@ -58,10 +47,7 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - type: bind - source: ./kibana/config/kibana.yml - target: /usr/share/kibana/config/kibana.yml - read_only: true + - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,z ports: - "5601:5601" networks: From 1815972838cef8c838eedffc6fb74672f6026abb Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 4 Dec 2021 00:41:45 +0100 Subject: [PATCH 23/55] Add Filebeat extension --- .github/workflows/ci.yml | 20 ++++++ .../workflows/scripts/run-tests-filebeat.sh | 61 +++++++++++++++++++ extensions/filebeat/Dockerfile | 3 + extensions/filebeat/README.md | 36 +++++++++++ extensions/filebeat/config/filebeat.yml | 30 +++++++++ extensions/filebeat/filebeat-compose.yml | 35 +++++++++++ extensions/metricbeat/metricbeat-compose.yml | 12 ++-- 7 files changed, 191 insertions(+), 6 deletions(-) create mode 100755 .github/workflows/scripts/run-tests-filebeat.sh create mode 100644 extensions/filebeat/Dockerfile create mode 100644 extensions/filebeat/README.md create mode 100644 extensions/filebeat/config/filebeat.yml create mode 100644 extensions/filebeat/filebeat-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a558020eb2..6d81b58c52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,7 @@ jobs: -f extensions/enterprise-search/enterprise-search-compose.yml \ -f extensions/apm-server/apm-server-compose.yml \ -f extensions/metricbeat/metricbeat-compose.yml \ + -f extensions/filebeat/filebeat-compose.yml \ build ######################################################## @@ -70,6 +71,7 @@ jobs: sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml + sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml # Run Elasticsearch and wait for its availability @@ -211,6 +213,23 @@ jobs: # next steps don't need Metricbeat docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat + # + # Filebeat + # + + - name: Execute Filebeat test suite + run: | + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up -d filebeat + .github/workflows/scripts/run-tests-filebeat.sh + + - name: 'debug: Display state and logs (Filebeat)' + if: always() + run: | + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml ps + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml logs filebeat + # next steps don't need Filebeat + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml stop filebeat + ############## # # # Tear down. # @@ -226,6 +245,7 @@ jobs: -f extensions/enterprise-search/enterprise-search-compose.yml -f extensions/apm-server/apm-server-compose.yml -f extensions/metricbeat/metricbeat-compose.yml + -f extensions/filebeat/filebeat-compose.yml down -v test-swarm: diff --git a/.github/workflows/scripts/run-tests-filebeat.sh b/.github/workflows/scripts/run-tests-filebeat.sh new file mode 100755 index 0000000000..567c59db26 --- /dev/null +++ b/.github/workflows/scripts/run-tests-filebeat.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + + +source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" + + +cid_es="$(container_id elasticsearch)" +cid_mb="$(container_id filebeat)" + +ip_es="$(service_ip elasticsearch)" +ip_mb="$(service_ip filebeat)" + +log 'Waiting for readiness of Elasticsearch' +poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' + +log 'Waiting for readiness of Filebeat' +poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" + +# We expect to find log entries for the 'elasticsearch' Compose service using +# the following query: +# +# agent.type:"filebeat" +# AND input.type:"container" +# AND container.name:"docker-elk-elasticsearch-1" +# +log 'Searching documents generated by Filebeat' + +declare response +declare -i count + +declare -i was_retried=0 + +# retry for max 60s (30*2s) +for _ in $(seq 1 30); do + response="$(curl "http://${ip_es}:9200/filebeat-*/_search?q=agent.type:%22filebeat%22%20AND%20input.type:%22container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)" + + set +u # prevent "unbound variable" if assigned value is not an integer + count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" + set -u + + if (( count > 0 )); then + break + fi + + was_retried=1 + echo -n 'x' >&2 + sleep 2 +done +if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 +fi + +echo "$response" +if (( count == 0 )); then + echo 'Expected at least 1 document' + exit 1 +fi diff --git a/extensions/filebeat/Dockerfile b/extensions/filebeat/Dockerfile new file mode 100644 index 0000000000..b70aa10308 --- /dev/null +++ b/extensions/filebeat/Dockerfile @@ -0,0 +1,3 @@ +ARG ELK_VERSION + +FROM docker.elastic.co/beats/filebeat:${ELK_VERSION} diff --git a/extensions/filebeat/README.md b/extensions/filebeat/README.md new file mode 100644 index 0000000000..43c5d67634 --- /dev/null +++ b/extensions/filebeat/README.md @@ -0,0 +1,36 @@ +# Filebeat + +Filebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers, +Filebeat monitors the log files or locations that you specify, collects log events, and forwards them either to +Elasticsearch or Logstash for indexing. + +## Usage + +To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line +argument referencing the `filebeat-compose.yml` file: + +```console +$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up +``` + +## Configuring Filebeat + +The Filebeat configuration is stored in [`config/filebeat.yml`](./config/filebeat.yml). You can modify this file with +the help of the [Configuration reference][filebeat-config]. + +Any change to the Filebeat configuration requires a restart of the Filebeat container: + +```console +$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat +``` + +Please refer to the following documentation page for more details about how to configure Filebeat inside a Docker +container: [Run Filebeat on Docker][filebeat-docker]. + +## See also + +[Filebeat documentation][filebeat-doc] + +[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html +[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html +[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html diff --git a/extensions/filebeat/config/filebeat.yml b/extensions/filebeat/config/filebeat.yml new file mode 100644 index 0000000000..347d070111 --- /dev/null +++ b/extensions/filebeat/config/filebeat.yml @@ -0,0 +1,30 @@ +## Filebeat configuration +## https://github.com/elastic/beats/blob/master/deploy/docker/filebeat.docker.yml +# + +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + # The Docker autodiscover provider automatically retrieves logs from Docker + # containers as they start and stop. + - type: docker + hints.enabled: true + +processors: + - add_cloud_metadata: ~ + +output.elasticsearch: + hosts: ['http://elasticsearch:9200'] + username: elastic + password: changeme + +## HTTP endpoint for health checking +## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html +# + +http.enabled: true +http.host: 0.0.0.0 diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml new file mode 100644 index 0000000000..40b115064b --- /dev/null +++ b/extensions/filebeat/filebeat-compose.yml @@ -0,0 +1,35 @@ +version: '3.2' + +services: + filebeat: + build: + context: extensions/filebeat/ + args: + ELK_VERSION: $ELK_VERSION + # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading + # 'docker.sock' and the host's filesystem. + user: root + command: + # Log to stderr. + - -e + # Disable config file permissions checks. Allows mounting + # 'config/filebeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + - --strict.perms=false + volumes: + - type: bind + source: ./extensions/filebeat/config/filebeat.yml + target: /usr/share/filebeat/filebeat.yml + read_only: true + - type: bind + source: /var/lib/docker/containers + target: /var/lib/docker/containers + read_only: true + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: true + networks: + - elk + depends_on: + - elasticsearch diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index c8f903225d..81ceb352ea 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -10,14 +10,14 @@ services: # 'docker.sock' and the host's filesystem. user: root command: - # Log to stderr. + # Log to stderr. - -e - # Disable config file permissions checks. Allows mounting - # 'config/metricbeat.yml' even if it's not owned by root. - # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + # Disable config file permissions checks. Allows mounting + # 'config/metricbeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html - --strict.perms=false - # Mount point of the host’s filesystem. Required to monitor the host - # from within a container. + # Mount point of the host’s filesystem. Required to monitor the host + # from within a container. - --system.hostfs=/hostfs volumes: - type: bind From 4e6c8712fe0ad35b73ac068ca780b10dbeae0dfe Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Sun, 12 Dec 2021 11:30:14 +0000 Subject: [PATCH 24/55] Update to v7.16.0 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 753d5828ac..b7fa18bccc 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.15.2 +ELK_VERSION=7.16.0 diff --git a/README.md b/README.md index dd43be9076..00377e65b8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -248,7 +248,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.15.2' \ + -H 'kbn-version: 7.16.0' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index 19919b42eb..33eeaa1a82 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.0 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.15.2 + image: docker.elastic.co/logstash/logstash:7.16.0 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.15.2 + image: docker.elastic.co/kibana/kibana:7.16.0 ports: - "5601:5601" configs: From c7f4a8eac0eb4ab787a76be6423a3b33d7764d49 Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 16:18:54 +0000 Subject: [PATCH 25/55] Update to v7.16.1 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index b7fa18bccc..89f25625f0 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.16.0 +ELK_VERSION=7.16.1 diff --git a/README.md b/README.md index 00377e65b8..dd579e7cc3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -248,7 +248,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.16.0' \ + -H 'kbn-version: 7.16.1' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index 33eeaa1a82..3693701ed8 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.16.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.16.0 + image: docker.elastic.co/logstash/logstash:7.16.1 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.16.0 + image: docker.elastic.co/kibana/kibana:7.16.1 ports: - "5601:5601" configs: From 691ffd8764fe22bc363836219b3ee2e089c2c12e Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Mon, 20 Dec 2021 16:01:32 +0000 Subject: [PATCH 26/55] Update to v7.16.2 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 89f25625f0..7c3b483373 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.16.1 +ELK_VERSION=7.16.2 diff --git a/README.md b/README.md index dd579e7cc3..6ec49f95c5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -248,7 +248,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.16.1' \ + -H 'kbn-version: 7.16.2' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index 3693701ed8..d214e5e86a 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.16.1 + image: docker.elastic.co/logstash/logstash:7.16.2 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.16.1 + image: docker.elastic.co/kibana/kibana:7.16.2 ports: - "5601:5601" configs: From 3ce7fc0ae342c1f957c1426c4ab618b00a0b484f Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Tue, 28 Dec 2021 20:16:54 +0100 Subject: [PATCH 27/55] ci: Fix race condition while checking container IP Be resilient and recheck the container's status if its IP address couldn't be determined on the first attempt. --- .github/workflows/scripts/lib/testing.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/lib/testing.sh b/.github/workflows/scripts/lib/testing.sh index 1099d1eef0..a217927624 100755 --- a/.github/workflows/scripts/lib/testing.sh +++ b/.github/workflows/scripts/lib/testing.sh @@ -74,7 +74,26 @@ function service_ip { local cid cid="$(container_id "$svc")" - ip="$(docker container inspect "$cid" --format '{{ (index .NetworkSettings.Networks "docker-elk_elk").IPAddress }}')" + local ip + + local -i was_retried=0 + + # retry for max 10s (5*2s) + for _ in $(seq 1 5); do + ip="$(docker container inspect "$cid" --format '{{ (index .NetworkSettings.Networks "docker-elk_elk").IPAddress }}')" + if [ -n "$ip" ]; then + break + fi + + was_retried=1 + echo -n '.' >&2 + sleep 2 + done + if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 + fi + if [ -z "${ip:-}" ]; then err "Container ${cid} has no IP address" return 1 From ff3506483637fd62dbb57487b0e8ab1498ab1c9d Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Sun, 16 Jan 2022 00:20:34 +0000 Subject: [PATCH 28/55] Update to v7.16.3 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 7c3b483373..528e6e28d0 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.16.2 +ELK_VERSION=7.16.3 diff --git a/README.md b/README.md index 6ec49f95c5..0cb80cfc9a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.3-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -248,7 +248,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.16.2' \ + -H 'kbn-version: 7.16.3' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index d214e5e86a..bd9a746721 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.16.2 + image: docker.elastic.co/logstash/logstash:7.16.3 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.16.2 + image: docker.elastic.co/kibana/kibana:7.16.3 ports: - "5601:5601" configs: From 1a30af6e11f65323b0f3a75a4e4ab54861adc240 Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Fri, 4 Feb 2022 14:37:42 +0000 Subject: [PATCH 29/55] Update to v7.17.0 --- .env | 2 +- README.md | 4 ++-- docker-stack.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 528e6e28d0..e40c94c1c7 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.16.3 +ELK_VERSION=7.17.0 diff --git a/README.md b/README.md index 0cb80cfc9a..d91ab8712a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.16.3-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.17.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -248,7 +248,7 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.16.3' \ + -H 'kbn-version: 7.17.0' \ -u elastic: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` diff --git a/docker-stack.yml b/docker-stack.yml index bd9a746721..611f14d95a 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.16.3 + image: docker.elastic.co/logstash/logstash:7.17.0 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.16.3 + image: docker.elastic.co/kibana/kibana:7.17.0 ports: - "5601:5601" configs: From 3882ce97e147ebd6f3cd2b3c4ac9f3190f0a3909 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 4 Feb 2022 15:44:55 +0100 Subject: [PATCH 30/55] doc: Update links to Legacy APM Server Reference --- extensions/apm-server/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/apm-server/README.md b/extensions/apm-server/README.md index 55df2d5b84..ef4f34e619 100644 --- a/extensions/apm-server/README.md +++ b/extensions/apm-server/README.md @@ -52,5 +52,5 @@ Configuration settings for each supported language are available in the APM docu [Running APM Server on Docker][apm-docker] -[apm-agents]: https://www.elastic.co/guide/en/apm/get-started/current/components.html#_apm_agents -[apm-docker]: https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html +[apm-agents]: https://www.elastic.co/guide/en/apm/guide/current/components.html +[apm-docker]: https://www.elastic.co/guide/en/apm/guide/current/running-on-docker.html From 6704d9f1d7aa6b10c6d6ca67deeb0a3930835665 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 10 Feb 2022 17:19:04 +0100 Subject: [PATCH 31/55] Update to v8.0.0 (#544) List of changes impacting docker-elk: - [logstash]: The output to Elasticsearch is handled as a data stream. Starting with v8.0.0, the `elasticsearch` output for Logstash sends log data to a data stream instead of `logstash-*` indices by default. The name of the default data stream is `logs-generic-default`. docker-elk remains unopinionated and simply uses Elastic's defaults like it always has, so users who prefer to retain the old behaviour need to explicitly opt-out of data streams in their Logstash pipelines. Refs: - https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html - https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams - [logstash]: The (legacy) monitoring data collection is now disabled. This feature was deprecated since v7.9.0, and removed in v8.0.0. Ref: https://www.elastic.co/guide/en/logstash/current/monitoring-internal-collection-legacy.html - [kibana]: An index pattern for `logs-*` indices is automatically created. It used to be required to manually create an index pattern for indices managed by Logstash, even when using the default Logstash indices. This is no longer the case since the output data is now being handled as a data stream, and Kibana automatically creates index patterns for these. - [elasticsearch]: The command line tool `elasticsearch-setup-passwords` was deprecated in favour of a new `elasticsearch-reset-password` tool. Passwords for built-in users must now be generated one by one. Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-passwords.html - [enterprise-search]: Kibana is now the new management interface, and the only one available moving forward. The old standalone Enterprise Search interface was removed in v8.0.0. Ref: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html --- .env | 2 +- .../scripts/elasticsearch-setup-passwords.exp | 32 ++++++--- .github/workflows/scripts/run-tests-core.sh | 26 ++----- .../workflows/scripts/run-tests-logspout.sh | 2 +- .github/workflows/update.yml | 8 +-- README.md | 72 ++++++++----------- docker-stack.yml | 6 +- extensions/enterprise-search/README.md | 13 ++++ .../config/enterprise-search.yml | 3 +- logstash/config/logstash.yml | 7 -- logstash/pipeline/logstash.conf | 1 - 11 files changed, 79 insertions(+), 93 deletions(-) diff --git a/.env b/.env index e40c94c1c7..4279128d06 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.17.0 +ELK_VERSION=8.0.0 diff --git a/.github/workflows/scripts/elasticsearch-setup-passwords.exp b/.github/workflows/scripts/elasticsearch-setup-passwords.exp index 3061f92b36..d2f9a36020 100755 --- a/.github/workflows/scripts/elasticsearch-setup-passwords.exp +++ b/.github/workflows/scripts/elasticsearch-setup-passwords.exp @@ -1,7 +1,7 @@ #!/usr/bin/expect -f # List of expected users with dummy password -set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)" +set users {"elastic" "kibana_system" "logstash_system" "beats_system" "apm_system" "remote_monitoring_user"} set password "testpasswd" # Find elasticsearch container id @@ -12,17 +12,27 @@ if { [string match "swarm" $MODE] } { set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch] } -set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b -u http://localhost:9200" +foreach user $users { + set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i" -spawn {*}$cmd + spawn {*}$cmd -expect { - -re "(E|Ree)nter password for \\\[$user\\\]: " { - send "$password\r" - exp_continue + expect { + -re "(E|Re-e)nter password for \\\[$user\\\]: " { + send "$password\r" + exp_continue + } + timeout { + puts "\ntimed out waiting for input" + exit 4 + } + eof } - eof -} -lassign [wait] pid spawnid os_error_flag value -exit $value + lassign [wait] pid spawnid os_error_flag value + + if {$value != 0} { + if {$os_error_flag == 0} { puts "exit status: $value" } else { puts "errno: $value" } + exit $value + } +} diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index 45a25052a0..51f9664713 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -24,25 +24,6 @@ poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty" log 'Waiting for readiness of Kibana' poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" -u 'kibana_system:testpasswd' -log 'Creating Logstash index pattern in Kibana' -source .env -curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \ - -s -w '\n' \ - -H 'Content-Type: application/json' \ - -H "kbn-version: ${ELK_VERSION}" \ - -u elastic:testpasswd \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' - -log 'Searching index pattern via Kibana API' -response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" -echo "$response" -declare -i count -count="$(jq -rn --argjson data "${response}" '$data.total')" -if (( count != 1 )); then - echo "Expected 1 index pattern, got ${count}" - exit 1 -fi - log 'Sending message to Logstash TCP input' declare -i was_retried=0 @@ -62,13 +43,14 @@ if ((was_retried)); then echo >&2 fi -sleep 3 -curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \ +sleep 5 +curl -X POST "http://${ip_es}:9200/logs-generic-default/_refresh" -u elastic:testpasswd \ -s -w '\n' log 'Searching message in Elasticsearch' -response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" +response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" echo "$response" +declare -i count count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" if (( count != 1 )); then echo "Expected 1 document, got ${count}" diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh index caf62bcbe3..ea1748b110 100755 --- a/.github/workflows/scripts/run-tests-logspout.sh +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -39,7 +39,7 @@ declare -i was_retried=0 # retry for max 60s (30*2s) for _ in $(seq 1 30); do - response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" + response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" set +u # prevent "unbound variable" if assigned value is not an integer count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index a22cfc6e87..6c7ac2954f 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -12,13 +12,13 @@ jobs: strategy: matrix: release: + - 8.x - 7.x - - 6.x include: - - release: 7.x + - release: 8.x branch: main - - release: 6.x - branch: release-6.x + - release: 7.x + branch: release-7.x steps: - uses: actions/setup-node@v2 diff --git a/README.md b/README.md index d91ab8712a..1860a1175a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.17.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -24,7 +24,7 @@ Based on the official Docker images from Elastic: Other available stack variants: -* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch. +* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch * [`searchguard`](https://github.com/deviantony/docker-elk/tree/searchguard): Search Guard support --- @@ -54,7 +54,6 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [Initial setup](#initial-setup) * [Setting up user authentication](#setting-up-user-authentication) * [Injecting data](#injecting-data) - * [Default Kibana index pattern creation](#default-kibana-index-pattern-creation) 1. [Configuration](#configuration) * [How to configure Elasticsearch](#how-to-configure-elasticsearch) * [How to configure Kibana](#how-to-configure-kibana) @@ -114,7 +113,7 @@ instructions from the [documentation][mac-filesharing] to add more locations. ### Version selection This repository tries to stay aligned with the latest version of the Elastic stack. The `main` branch tracks the current -major version (7.x). +major version (8.x). To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If you are upgrading an existing stack, please carefully read the note in the next section. @@ -124,8 +123,9 @@ performing a stack upgrade.** Older major versions are also supported on separate branches: -* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series -* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-Of-Life) +* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series +* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life) +* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life) ### Bringing up the stack @@ -168,11 +168,31 @@ users][builtin-users] instead for increased security. 1. Initialize passwords for built-in users + The commands below generate random passwords for all 6 built-in users. Take note of them. + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user elastic + ``` + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system + ``` + ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-setup-passwords auto --batch + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user logstash_system ``` - Passwords for all 6 built-in users will be randomly generated. Take note of them. + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user beats_system + ``` + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user apm_system + ``` + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user remote_monitoring_user + ``` 1. Unset the bootstrap password (_optional_) @@ -181,9 +201,8 @@ users][builtin-users] instead for increased security. 1. Replace usernames and passwords in configuration files - Use the `kibana_system` user (`kibana` for releases <7.8.0) inside the Kibana configuration file - (`kibana/config/kibana.yml`) and the `logstash_system` user inside the Logstash configuration file - (`logstash/config/logstash.yml`) in place of the existing `elastic` user. + Use the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) in place of the + existing `elastic` user. Replace the password for the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`). @@ -225,37 +244,6 @@ $ cat /path/to/logfile.log | nc -c localhost 5000 You can also load the sample data provided by your Kibana installation. -### Default Kibana index pattern creation - -When Kibana launches for the first time, it is not configured with any index pattern. - -#### Via the Kibana web UI - -*:information_source: You need to inject data into Logstash before being able to configure a Logstash index pattern via -the Kibana web UI.* - -Navigate to the _Discover_ view of Kibana from the left sidebar. You will be prompted to create an index pattern. Enter -`logstash-*` to match Logstash indices then, on the next page, select `@timestamp` as the time filter field. Finally, -click _Create index pattern_ and return to the _Discover_ view to inspect your log entries. - -Refer to [Connect Kibana with Elasticsearch][connect-kibana] and [Creating an index pattern][index-pattern] for detailed -instructions about the index pattern configuration. - -#### On the command line - -Create an index pattern via the Kibana API: - -```console -$ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ - -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.17.0' \ - -u elastic: \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' -``` - -The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the -first time. - ## Configuration *:information_source: Configuration is not dynamically reloaded, you will need to restart individual components after diff --git a/docker-stack.yml b/docker-stack.yml index 611f14d95a..f7eaad8f27 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.17.0 + image: docker.elastic.co/logstash/logstash:8.0.0 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.17.0 + image: docker.elastic.co/kibana/kibana:8.0.0 ports: - "5601:5601" configs: diff --git a/extensions/enterprise-search/README.md b/extensions/enterprise-search/README.md index d055c8852f..71fd789dec 100644 --- a/extensions/enterprise-search/README.md +++ b/extensions/enterprise-search/README.md @@ -57,6 +57,17 @@ add the following setting: xpack.security.authc.api_key.enabled: true ``` +### Configure the Enterprise Search host in Kibana + +Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search. + +To enable the management experience for Enterprise Search, modify the Kibana configuration file in +[`kibana/config/kibana.yml`][config-kbn] and add the following setting: + +```yaml +enterpriseSearch.host: http://enterprise-search:3002 +``` + ### Start the server To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command @@ -129,6 +140,8 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock [enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html [enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html [enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html +[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html [es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings [config-es]: ../../elasticsearch/config/elasticsearch.yml +[config-kbn]: ../../kibana/config/kibana.yml diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index 891b510a53..eb94457a72 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -15,8 +15,9 @@ secret_management.encryption_keys: # IP address Enterprise Search listens on ent_search.listen_host: 0.0.0.0 -# URL at which users reach Enterprise Search +# URL at which users reach Enterprise Search / Kibana ent_search.external_url: http://localhost:3002 +kibana.host: http://localhost:5601 # Elasticsearch URL and credentials elasticsearch.host: http://elasticsearch:9200 diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index a48c35ff58..47722ea7f3 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -3,10 +3,3 @@ ## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml # http.host: "0.0.0.0" -xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] - -## X-Pack security credentials -# -xpack.monitoring.enabled: true -xpack.monitoring.elasticsearch.username: elastic -xpack.monitoring.elasticsearch.password: changeme diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf index 7d5918ba88..40ca75775c 100644 --- a/logstash/pipeline/logstash.conf +++ b/logstash/pipeline/logstash.conf @@ -15,6 +15,5 @@ output { hosts => "elasticsearch:9200" user => "elastic" password => "changeme" - ecs_compatibility => disabled } } From 33a96e571a816b3417542496de16bbb2dfeb5afe Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 10 Feb 2022 17:56:20 +0100 Subject: [PATCH 32/55] es: Remove deprecated monitoring setting --- elasticsearch/config/elasticsearch.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml index 86822dd077..3e82379941 100644 --- a/elasticsearch/config/elasticsearch.yml +++ b/elasticsearch/config/elasticsearch.yml @@ -6,8 +6,7 @@ cluster.name: "docker-cluster" network.host: 0.0.0.0 ## X-Pack settings -## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html # xpack.license.self_generated.type: trial xpack.security.enabled: true -xpack.monitoring.collection.enabled: true From 370ebbfaf41afd67f1533d0a66602641aa98e5e2 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 11 Feb 2022 14:29:01 +0100 Subject: [PATCH 33/55] doc: Update "Initial Setup" section for v8.0.0 Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged `elastic` user. Closes #666 --- .github/workflows/ci.yml | 4 +- README.md | 138 +++++++++++++++++++-------------------- kibana/config/kibana.yml | 2 +- 3 files changed, 71 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d81b58c52..8c427315d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf - sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana_system/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml + sed -i 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml @@ -288,7 +288,7 @@ jobs: sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf - sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana_system/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml + sed -i 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml # Run Elasticsearch and wait for its availability diff --git a/README.md b/README.md index 1860a1175a..088d76ba4c 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,11 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [Windows](#windows) * [macOS](#macos) 1. [Usage](#usage) - * [Version selection](#version-selection) - * [Bringing up the stack](#bringing-up-the-stack) - * [Cleanup](#cleanup) * [Initial setup](#initial-setup) * [Setting up user authentication](#setting-up-user-authentication) * [Injecting data](#injecting-data) + * [Cleanup](#cleanup) + * [Version selection](#version-selection) 1. [Configuration](#configuration) * [How to configure Elasticsearch](#how-to-configure-elasticsearch) * [How to configure Kibana](#how-to-configure-kibana) @@ -110,65 +109,40 @@ instructions from the [documentation][mac-filesharing] to add more locations. ## Usage -### Version selection - -This repository tries to stay aligned with the latest version of the Elastic stack. The `main` branch tracks the current -major version (8.x). - -To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If -you are upgrading an existing stack, please carefully read the note in the next section. - -**:warning: Always pay attention to the [official upgrade instructions][upgrade] for each individual component before -performing a stack upgrade.** - -Older major versions are also supported on separate branches: - -* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series -* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life) -* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life) - -### Bringing up the stack - -Clone this repository onto the Docker host that will run the stack, then start services locally using Docker Compose: - -```console -$ docker-compose up -``` - -You can also run all services in the background (detached mode) by adding the `-d` flag to the above command. - **:warning: You must rebuild the stack images with `docker-compose build` whenever you switch branch or update the -version of an already existing stack.** +[version](#version-selection) of an already existing stack.** -If you are starting the stack for the very first time, please read the section below attentively. +### Initial setup -### Cleanup - -Elasticsearch data is persisted inside a volume by default. - -In order to entirely shutdown the stack and remove all persisted data, use the following Docker Compose command: +Clone this repository onto the Docker host that will run the stack, then start the Elasticsearch service locally using +Docker Compose: ```console -$ docker-compose down -v +$ docker-compose up elasticsearch -d ``` -## Initial setup +We will start the rest of the Elastic components _after_ completing the initial setup described in this section. These +steps only need to be performed _once_. + +**:warning: Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged +`elastic` user. If you are starting the stack for the very first time, you MUST initialize a password for the [built-in +`kibana_system` user][builtin-users] to be able to start and access Kibana. Please read the section below attentively.** -### Setting up user authentication +#### Setting up user authentication -*:information_source: Refer to [How to disable paid features](#how-to-disable-paid-features) to disable authentication.* +*:information_source: Refer to [Security settings in Elasticsearch][es-security] to disable authentication.* The stack is pre-configured with the following **privileged** bootstrap user: * user: *elastic* * password: *changeme* -Although all stack components work out-of-the-box with this user, we strongly recommend using the unprivileged [built-in -users][builtin-users] instead for increased security. +For increased security, we will reset this bootstrap password, and generate a set of passwords to be used by +unprivileged [built-in users][builtin-users] within components of the Elastic stack. 1. Initialize passwords for built-in users - The commands below generate random passwords for all 6 built-in users. Take note of them. + The commands below generate random passwords for the `elastic` and `kibana_system` users. Take note of them. ```console $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user elastic @@ -178,33 +152,17 @@ users][builtin-users] instead for increased security. $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system ``` - ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user logstash_system - ``` - - ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user beats_system - ``` - - ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user apm_system - ``` - - ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user remote_monitoring_user - ``` - -1. Unset the bootstrap password (_optional_) - - Remove the `ELASTIC_PASSWORD` environment variable from the `elasticsearch` service inside the Compose file - (`docker-compose.yml`). It is only used to initialize the keystore during the initial startup of Elasticsearch. + If the need for it arises (e.g. if you want to [collect monitoring information][ls-monitoring] through Beats and + other components), feel free to repeat this operation at any time for the rest of the [built-in + users][builtin-users]. 1. Replace usernames and passwords in configuration files - Use the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) in place of the - existing `elastic` user. + Replace the password of the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) + with the password generated in the previous step. - Replace the password for the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`). + Replace the password of the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`) + with the password generated in the previous step. *:information_source: Do not use the `logstash_system` user inside the Logstash **pipeline** file, it does not have sufficient permissions to create indices. Follow the instructions at [Configuring Security in Logstash][ls-security] @@ -212,16 +170,27 @@ users][builtin-users] instead for increased security. See also the [Configuration](#configuration) section below. -1. Restart Kibana and Logstash to apply changes +1. Unset the bootstrap password (_optional_) + + Remove the `ELASTIC_PASSWORD` environment variable from the `elasticsearch` service inside the Compose file + (`docker-compose.yml`). It is only used to initialize the keystore during the initial startup of Elasticsearch, and + is ignored on subsequent runs. + +1. Start Kibana and Logstash ```console - $ docker-compose restart kibana logstash + $ docker-compose up -d ``` + The `-d` flag runs all services in the background (detached mode). + + On subsequent runs of the Elastic stack, it is sufficient to execute the above command in order to start all + components. + *:information_source: Learn more about the security of the Elastic stack at [Secure the Elastic Stack][sec-cluster].* -### Injecting data +#### Injecting data Give Kibana about a minute to initialize, then access the Kibana web UI by opening in a web browser and use the following credentials to log in: @@ -244,6 +213,33 @@ $ cat /path/to/logfile.log | nc -c localhost 5000 You can also load the sample data provided by your Kibana installation. +### Cleanup + +Elasticsearch data is persisted inside a volume by default. + +In order to entirely shutdown the stack and remove all persisted data, use the following Docker Compose command: + +```console +$ docker-compose down -v +``` + +### Version selection + +This repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major +version (8.x). + +To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If +you are upgrading an existing stack, please carefully read the note in the next section. + +**:warning: Always pay attention to the [official upgrade instructions][upgrade] for each individual component before +performing a stack upgrade.** + +Older major versions are also supported on separate branches: + +* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series +* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life) +* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life) + ## Configuration *:information_source: Configuration is not dynamically reloaded, you will need to restart individual components after @@ -404,6 +400,7 @@ instead of `elasticsearch`.* [elk-stack]: https://www.elastic.co/what-is/elk-stack [xpack]: https://www.elastic.co/what-is/open-x-pack [paid-features]: https://www.elastic.co/subscriptions +[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html [trial-license]: https://www.elastic.co/guide/en/elasticsearch/reference/current/license-settings.html [license-mngmt]: https://www.elastic.co/guide/en/kibana/current/managing-licenses.html [license-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html @@ -420,6 +417,7 @@ instead of `elasticsearch`.* [builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html [ls-security]: https://www.elastic.co/guide/en/logstash/current/ls-security.html +[ls-monitoring]: https://www.elastic.co/guide/en/logstash/current/monitoring-with-metricbeat.html [sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-cluster.html [connect-kibana]: https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 0e1dc60c54..456b3f7481 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -9,5 +9,5 @@ monitoring.ui.container.elasticsearch.enabled: true ## X-Pack security credentials # -elasticsearch.username: elastic +elasticsearch.username: kibana_system elasticsearch.password: changeme From b77b7fb66a1fca7521da6e3447da4ae7f5ba89ca Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 11 Feb 2022 19:10:26 +0100 Subject: [PATCH 34/55] fix: Swap flag with service name in Compose command Related to #667 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 088d76ba4c..8cfb078468 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Clone this repository onto the Docker host that will run the stack, then start t Docker Compose: ```console -$ docker-compose up elasticsearch -d +$ docker-compose up -d elasticsearch ``` We will start the rest of the Elastic components _after_ completing the initial setup described in this section. These From ca98168da1f8b08ee0d3eb16d8909ef50ff0454b Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 11 Feb 2022 19:06:13 +0100 Subject: [PATCH 35/55] Quote passwords to prevent interpreting leading special chars Closes #667 --- .github/workflows/ci.yml | 14 ++++++-------- docker-compose.yml | 6 +++--- extensions/apm-server/config/apm-server.yml | 2 +- extensions/curator/config/curator.yml | 2 +- .../enterprise-search/config/enterprise-search.yml | 2 +- .../enterprise-search-compose.yml | 2 +- extensions/filebeat/config/filebeat.yml | 2 +- extensions/metricbeat/config/metricbeat.yml | 2 +- kibana/config/kibana.yml | 2 +- 9 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c427315d5..8c5d5d041b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,13 +65,12 @@ jobs: # Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files - sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf - sed -i 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml - sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml - sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml - sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml - sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml + sed -i 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' kibana/config/kibana.yml + sed -i -e 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml + sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml + sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml + sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml # Run Elasticsearch and wait for its availability @@ -286,9 +285,8 @@ jobs: # Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files - sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf - sed -i 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml + sed -i 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' kibana/config/kibana.yml # Run Elasticsearch and wait for its availability diff --git a/docker-compose.yml b/docker-compose.yml index c7429e5c83..91dc17de4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,8 @@ services: - "9200:9200" - "9300:9300" environment: - ES_JAVA_OPTS: "-Xmx256m -Xms256m" - ELASTIC_PASSWORD: changeme + ES_JAVA_OPTS: -Xmx256m -Xms256m + ELASTIC_PASSWORD: 'changeme' # Use single node discovery in order to disable production mode and avoid bootstrap checks. # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html discovery.type: single-node @@ -35,7 +35,7 @@ services: - "5000:5000/udp" - "9600:9600" environment: - LS_JAVA_OPTS: "-Xmx256m -Xms256m" + LS_JAVA_OPTS: -Xmx256m -Xms256m networks: - elk depends_on: diff --git a/extensions/apm-server/config/apm-server.yml b/extensions/apm-server/config/apm-server.yml index 493a49e65b..e78ea7d0b1 100644 --- a/extensions/apm-server/config/apm-server.yml +++ b/extensions/apm-server/config/apm-server.yml @@ -5,4 +5,4 @@ output: elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: changeme + password: 'changeme' diff --git a/extensions/curator/config/curator.yml b/extensions/curator/config/curator.yml index 8b06971732..f4cf8de3a9 100644 --- a/extensions/curator/config/curator.yml +++ b/extensions/curator/config/curator.yml @@ -5,7 +5,7 @@ client: hosts: - elasticsearch port: 9200 - http_auth: elastic:changeme + http_auth: 'elastic:changeme' logging: loglevel: INFO diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index eb94457a72..49947c412a 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -22,7 +22,7 @@ kibana.host: http://localhost:5601 # Elasticsearch URL and credentials elasticsearch.host: http://elasticsearch:9200 elasticsearch.username: elastic -elasticsearch.password: changeme +elasticsearch.password: 'changeme' # Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes. allow_es_settings_modification: true diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml index cd58caa6f3..dd57424365 100644 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ b/extensions/enterprise-search/enterprise-search-compose.yml @@ -13,7 +13,7 @@ services: read_only: true environment: JAVA_OPTS: -Xmx2g -Xms2g - ENT_SEARCH_DEFAULT_PASSWORD: changeme + ENT_SEARCH_DEFAULT_PASSWORD: 'changeme' ports: - '3002:3002' networks: diff --git a/extensions/filebeat/config/filebeat.yml b/extensions/filebeat/config/filebeat.yml index 347d070111..18cf5f643c 100644 --- a/extensions/filebeat/config/filebeat.yml +++ b/extensions/filebeat/config/filebeat.yml @@ -20,7 +20,7 @@ processors: output.elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: changeme + password: 'changeme' ## HTTP endpoint for health checking ## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html diff --git a/extensions/metricbeat/config/metricbeat.yml b/extensions/metricbeat/config/metricbeat.yml index eac94d6d27..3294035d82 100644 --- a/extensions/metricbeat/config/metricbeat.yml +++ b/extensions/metricbeat/config/metricbeat.yml @@ -34,7 +34,7 @@ processors: output.elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: changeme + password: 'changeme' ## HTTP endpoint for health checking ## https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 456b3f7481..9025104d5b 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -10,4 +10,4 @@ monitoring.ui.container.elasticsearch.enabled: true ## X-Pack security credentials # elasticsearch.username: kibana_system -elasticsearch.password: changeme +elasticsearch.password: 'changeme' From a82e95d94edcd82d6cc6fe9acf1e84164473fcb5 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 11 Feb 2022 19:51:43 +0100 Subject: [PATCH 36/55] doc: Provide examples of config overrides via env vars --- README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8cfb078468..a4221768d3 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,14 @@ containers: [Install Elasticsearch with Docker][es-docker]. The Kibana default configuration is stored in [`kibana/config/kibana.yml`][config-kbn]. -It is also possible to map the entire `config` directory instead of a single file. +You can also specify the options you want to override by setting environment variables inside the Compose file: + +```yml +kibana: + + environment: + SERVER_NAME: kibana.example.org +``` Please refer to the following documentation page for more details about how to configure Kibana inside Docker containers: [Install Kibana with Docker][kbn-docker]. @@ -275,8 +282,14 @@ containers: [Install Kibana with Docker][kbn-docker]. The Logstash configuration is stored in [`logstash/config/logstash.yml`][config-ls]. -It is also possible to map the entire `config` directory instead of a single file, however you must be aware that -Logstash will be expecting a [`log4j2.properties`][log4j-props] file for its own logging. +You can also specify the options you want to override by setting environment variables inside the Compose file: + +```yml +logstash: + + environment: + LOG_LEVEL: debug +``` Please refer to the following documentation page for more details about how to configure Logstash inside Docker containers: [Configuring Logstash for Docker][ls-docker]. @@ -431,9 +444,6 @@ instead of `elasticsearch`.* [kbn-docker]: https://www.elastic.co/guide/en/kibana/current/docker.html [ls-docker]: https://www.elastic.co/guide/en/logstash/current/docker-config.html -[log4j-props]: https://github.com/elastic/logstash/tree/7.6/docker/data/logstash/config -[esuser]: https://github.com/elastic/elasticsearch/blob/7.6/distribution/docker/src/docker/Dockerfile#L23-L24 - [upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html [swarm-mode]: https://docs.docker.com/engine/swarm/ From 641290c20af46e995fa6ac7c8a0584b96b17c8a5 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 11 Feb 2022 20:11:22 +0100 Subject: [PATCH 37/55] Apply SELinux options to bind mounts https://docs.docker.com/storage/bind-mounts/ Related to #639 Closes #661 --- docker-compose.yml | 6 +++--- extensions/apm-server/apm-server-compose.yml | 5 +---- extensions/curator/curator-compose.yml | 10 ++-------- .../enterprise-search/enterprise-search-compose.yml | 5 +---- extensions/filebeat/filebeat-compose.yml | 5 +---- extensions/logspout/logspout-compose.yml | 5 ++++- extensions/metricbeat/metricbeat-compose.yml | 5 +---- 7 files changed, 13 insertions(+), 28 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 91dc17de4d..3411d1285c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,8 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,z - - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,z + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z + - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z ports: - "5044:5044" - "5000:5000/tcp" @@ -47,7 +47,7 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,z + - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z ports: - "5601:5601" networks: diff --git a/extensions/apm-server/apm-server-compose.yml b/extensions/apm-server/apm-server-compose.yml index bd21a2d502..991e553255 100644 --- a/extensions/apm-server/apm-server-compose.yml +++ b/extensions/apm-server/apm-server-compose.yml @@ -11,10 +11,7 @@ services: # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html - --strict.perms=false volumes: - - type: bind - source: ./extensions/apm-server/config/apm-server.yml - target: /usr/share/apm-server/apm-server.yml - read_only: true + - ./extensions/apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro,Z ports: - '8200:8200' networks: diff --git a/extensions/curator/curator-compose.yml b/extensions/curator/curator-compose.yml index 96dae3cc9a..60786a6628 100644 --- a/extensions/curator/curator-compose.yml +++ b/extensions/curator/curator-compose.yml @@ -6,14 +6,8 @@ services: context: extensions/curator/ init: true volumes: - - type: bind - source: ./extensions/curator/config/curator.yml - target: /usr/share/curator/config/curator.yml - read_only: true - - type: bind - source: ./extensions/curator/config/delete_log_files_curator.yml - target: /usr/share/curator/config/delete_log_files_curator.yml - read_only: true + - ./extensions/curator/config/curator.yml:/usr/share/curator/config/curator.yml:ro,Z + - ./extensions/curator/config/delete_log_files_curator.yml:/usr/share/curator/config/delete_log_files_curator.yml:ro,Z networks: - elk depends_on: diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml index dd57424365..fcb12c14d1 100644 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ b/extensions/enterprise-search/enterprise-search-compose.yml @@ -7,10 +7,7 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - type: bind - source: ./extensions/enterprise-search/config/enterprise-search.yml - target: /usr/share/enterprise-search/config/enterprise-search.yml - read_only: true + - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z environment: JAVA_OPTS: -Xmx2g -Xms2g ENT_SEARCH_DEFAULT_PASSWORD: 'changeme' diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml index 40b115064b..86dd11daa2 100644 --- a/extensions/filebeat/filebeat-compose.yml +++ b/extensions/filebeat/filebeat-compose.yml @@ -17,10 +17,7 @@ services: # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html - --strict.perms=false volumes: - - type: bind - source: ./extensions/filebeat/config/filebeat.yml - target: /usr/share/filebeat/filebeat.yml - read_only: true + - ./extensions/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,Z - type: bind source: /var/lib/docker/containers target: /var/lib/docker/containers diff --git a/extensions/logspout/logspout-compose.yml b/extensions/logspout/logspout-compose.yml index 1c349f22b4..bb60bef018 100644 --- a/extensions/logspout/logspout-compose.yml +++ b/extensions/logspout/logspout-compose.yml @@ -5,7 +5,10 @@ services: build: context: extensions/logspout volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: true environment: ROUTE_URIS: logstash://logstash:5000 LOGSTASH_TAGS: docker-elk diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index 81ceb352ea..24770ed43a 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -20,10 +20,7 @@ services: # from within a container. - --system.hostfs=/hostfs volumes: - - type: bind - source: ./extensions/metricbeat/config/metricbeat.yml - target: /usr/share/metricbeat/metricbeat.yml - read_only: true + - ./extensions/metricbeat/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro,Z - type: bind source: / target: /hostfs From 9877b39900076a764036352b331a6115adc91c87 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 21 Feb 2022 11:19:43 +0100 Subject: [PATCH 38/55] Automate management of required roles and user passwords (#671) --- .dockerignore | 2 - .env | 21 ++ .github/workflows/ci.yml | 117 +---------- .../scripts/elasticsearch-setup-passwords.exp | 38 ---- .github/workflows/scripts/lib/testing.sh | 22 +-- .github/workflows/update.yml | 2 +- README.md | 121 +++++------- docker-compose.yml | 37 +++- docker-stack.yml | 72 ------- elasticsearch/.dockerignore | 6 + extensions/apm-server/.dockerignore | 6 + extensions/apm-server/apm-server-compose.yml | 4 +- extensions/apm-server/config/apm-server.yml | 2 +- extensions/curator/.dockerignore | 6 + extensions/enterprise-search/.dockerignore | 6 + .../config/enterprise-search.yml | 2 +- .../enterprise-search-compose.yml | 3 +- extensions/filebeat/.dockerignore | 6 + extensions/filebeat/config/filebeat.yml | 2 +- extensions/filebeat/filebeat-compose.yml | 4 +- extensions/logspout/.dockerignore | 6 + extensions/metricbeat/.dockerignore | 6 + extensions/metricbeat/config/metricbeat.yml | 2 +- extensions/metricbeat/metricbeat-compose.yml | 4 +- kibana/.dockerignore | 6 + kibana/config/kibana.yml | 2 +- logstash/.dockerignore | 6 + logstash/pipeline/logstash.conf | 4 +- setup/.dockerignore | 12 ++ setup/.gitignore | 1 + setup/Dockerfile | 11 ++ setup/entrypoint.sh | 85 ++++++++ setup/helpers.sh | 182 ++++++++++++++++++ setup/roles/logstash_writer.json | 22 +++ 34 files changed, 502 insertions(+), 326 deletions(-) delete mode 100644 .dockerignore delete mode 100755 .github/workflows/scripts/elasticsearch-setup-passwords.exp delete mode 100644 docker-stack.yml create mode 100644 elasticsearch/.dockerignore create mode 100644 extensions/apm-server/.dockerignore create mode 100644 extensions/curator/.dockerignore create mode 100644 extensions/enterprise-search/.dockerignore create mode 100644 extensions/filebeat/.dockerignore create mode 100644 extensions/logspout/.dockerignore create mode 100644 extensions/metricbeat/.dockerignore create mode 100644 kibana/.dockerignore create mode 100644 logstash/.dockerignore create mode 100644 setup/.dockerignore create mode 100644 setup/.gitignore create mode 100644 setup/Dockerfile create mode 100755 setup/entrypoint.sh create mode 100755 setup/helpers.sh create mode 100644 setup/roles/logstash_writer.json diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index e873dbbde0..0000000000 --- a/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# Ignore OS artifacts -**/.DS_Store diff --git a/.env b/.env index 4279128d06..b4197807bd 100644 --- a/.env +++ b/.env @@ -1 +1,22 @@ ELK_VERSION=8.0.0 + +## Passwords for stack users +# + +# User 'elastic' (built-in) +# +# Superuser role, full access to cluster management and data indices. +# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html +ELASTIC_PASSWORD='changeme' + +# User 'logstash_internal' (custom) +# +# The user Logstash uses to connect and send data to Elasticsearch. +# https://www.elastic.co/guide/en/logstash/current/ls-security.html +LOGSTASH_INTERNAL_PASSWORD='changeme' + +# User 'kibana_system' (built-in) +# +# The user Kibana uses to connect and communicate with Elasticsearch. +# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html +KIBANA_SYSTEM_PASSWORD='changeme' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c5d5d041b..b5fd2bfd2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,8 @@ on: jobs: - test-compose: - name: 'Test suite: Compose' + test: + name: Test suite # List of supported runners: # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources runs-on: ubuntu-latest @@ -26,13 +26,6 @@ jobs: - name: Prepare environment run: | - # Install Linux packages - # - # List of packages pre-installed in the runner: - # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-software - - sudo apt install -y expect - # Enable support for Compose V2 # # Instructions: @@ -63,24 +56,9 @@ jobs: - name: Set password of every built-in user to 'testpasswd' run: | - # Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files - - sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf - sed -i 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' kibana/config/kibana.yml - sed -i -e 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml - sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml - sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml - sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml - - # Run Elasticsearch and wait for its availability - - docker compose up -d elasticsearch - source .github/workflows/scripts/lib/testing.sh - poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme' - - # Set passwords - - .github/workflows/scripts/elasticsearch-setup-passwords.exp + sed -i -e 's/\(ELASTIC_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \ + -e 's/\(LOGSTASH_INTERNAL_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \ + -e 's/\(KIBANA_SYSTEM_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' .env ########################################################## # # @@ -100,6 +78,7 @@ jobs: if: always() run: | docker compose ps + docker compose logs setup docker compose logs elasticsearch docker compose logs logstash docker compose logs kibana @@ -162,6 +141,8 @@ jobs: # Run Enterprise Search and execute tests + sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml + docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -d enterprise-search .github/workflows/scripts/run-tests-enterprise-search.sh @@ -246,85 +227,3 @@ jobs: -f extensions/metricbeat/metricbeat-compose.yml -f extensions/filebeat/filebeat-compose.yml down -v - - test-swarm: - name: 'Test suite: Swarm' - runs-on: ubuntu-latest - - env: - MODE: swarm - - steps: - - uses: actions/checkout@v2 - - ##################################################### - # # - # Install all dependencies required by test suites. # - # # - ##################################################### - - - name: Prepare environment - run: | - - # Install Linux packages - - sudo apt install -y expect - - # Enable Swarm mode - - docker swarm init - - ######################################################## - # # - # Ensure §"Initial setup" of the README remains valid. # - # # - ######################################################## - - - name: Set password of every built-in user to 'testpasswd' - run: | - - # Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files - - sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf - sed -i 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' kibana/config/kibana.yml - - # Run Elasticsearch and wait for its availability - - docker stack deploy -c ./docker-stack.yml elk - docker service scale elk_logstash=0 elk_kibana=0 - source .github/workflows/scripts/lib/testing.sh - poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme' - - # Set passwords - - .github/workflows/scripts/elasticsearch-setup-passwords.exp swarm - - ########################################################## - # # - # Test core components: Elasticsearch, Logstash, Kibana. # - # # - ########################################################## - - - name: Run the stack - run: docker service scale elk_logstash=1 elk_kibana=1 - - - name: Execute core test suite - run: .github/workflows/scripts/run-tests-core.sh swarm - - - name: 'debug: Display state and logs (core)' - if: always() - run: | - docker stack services elk - docker service logs elk_elasticsearch - docker service logs elk_kibana - docker service logs elk_logstash - - ############## - # # - # Tear down. # - # # - ############## - - - name: Terminate all components - if: always() - run: docker stack rm elk diff --git a/.github/workflows/scripts/elasticsearch-setup-passwords.exp b/.github/workflows/scripts/elasticsearch-setup-passwords.exp deleted file mode 100755 index d2f9a36020..0000000000 --- a/.github/workflows/scripts/elasticsearch-setup-passwords.exp +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/expect -f - -# List of expected users with dummy password -set users {"elastic" "kibana_system" "logstash_system" "beats_system" "apm_system" "remote_monitoring_user"} -set password "testpasswd" - -# Find elasticsearch container id -set MODE [lindex $argv 0] -if { [string match "swarm" $MODE] } { - set cid [exec docker ps -q -f label=com.docker.swarm.service.name=elk_elasticsearch] -} else { - set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch] -} - -foreach user $users { - set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i" - - spawn {*}$cmd - - expect { - -re "(E|Re-e)nter password for \\\[$user\\\]: " { - send "$password\r" - exp_continue - } - timeout { - puts "\ntimed out waiting for input" - exit 4 - } - eof - } - - lassign [wait] pid spawnid os_error_flag value - - if {$value != 0} { - if {$os_error_flag == 0} { puts "exit status: $value" } else { puts "errno: $value" } - exit $value - } -} diff --git a/.github/workflows/scripts/lib/testing.sh b/.github/workflows/scripts/lib/testing.sh index a217927624..1c77954dcf 100755 --- a/.github/workflows/scripts/lib/testing.sh +++ b/.github/workflows/scripts/lib/testing.sh @@ -14,12 +14,7 @@ function err { function container_id { local svc=$1 - local label - if [[ "${MODE:-}" == "swarm" ]]; then - label="com.docker.swarm.service.name=elk_${svc}" - else - label="com.docker.compose.service=${svc}" - fi + local label="com.docker.compose.service=${svc}" local cid @@ -51,26 +46,11 @@ function container_id { # Return the IP address at which a service can be reached. # In Compose mode, returns the container's IP. -# In Swarm mode, returns the IP of the node to ensure traffic enters the routing mesh (ingress). function service_ip { local svc=$1 local ip - if [[ "${MODE:-}" == "swarm" ]]; then - #ingress_net="$(docker network inspect ingress --format '{{ .Id }}')" - #ip="$(docker service inspect elk_"$svc" --format "{{ range .Endpoint.VirtualIPs }}{{ if eq .NetworkID \"${ingress_net}\" }}{{ .Addr }}{{ end }}{{ end }}" | cut -d/ -f1)" - node="$(docker node ls --format '{{ .ID }}')" - ip="$(docker node inspect "$node" --format '{{ .Status.Addr }}')" - if [ -z "${ip:-}" ]; then - err "Node ${node} has no IP address" - return 1 - fi - - echo "$ip" - return - fi - local cid cid="$(container_id "$svc")" diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 6c7ac2954f..e0a0f03dab 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -71,7 +71,7 @@ jobs: # Escape dot characters so sed interprets them as literal dots cur_ver="$(echo $cur_ver | sed 's/\./\\./g')" - for f in .env docker-stack.yml README.md; do + for f in .env README.md; do sed -i "s/${cur_ver}/${new_ver}/g" "$f" done diff --git a/README.md b/README.md index a4221768d3..d0747dba42 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [Windows](#windows) * [macOS](#macos) 1. [Usage](#usage) + * [Bringing up the stack](#bringing-up-the-stack) * [Initial setup](#initial-setup) * [Setting up user authentication](#setting-up-user-authentication) * [Injecting data](#injecting-data) @@ -68,7 +69,6 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [How to enable a remote JMX connection to a service](#how-to-enable-a-remote-jmx-connection-to-a-service) 1. [Going further](#going-further) * [Plugins and integrations](#plugins-and-integrations) - * [Swarm mode](#swarm-mode) ## Requirements @@ -112,44 +112,57 @@ instructions from the [documentation][mac-filesharing] to add more locations. **:warning: You must rebuild the stack images with `docker-compose build` whenever you switch branch or update the [version](#version-selection) of an already existing stack.** -### Initial setup +### Bringing up the stack -Clone this repository onto the Docker host that will run the stack, then start the Elasticsearch service locally using -Docker Compose: +Clone this repository onto the Docker host that will run the stack, then start the stack's services locally using Docker +Compose: ```console -$ docker-compose up -d elasticsearch +$ docker-compose up ``` -We will start the rest of the Elastic components _after_ completing the initial setup described in this section. These -steps only need to be performed _once_. +*:information_source: You can also run all services in the background (detached mode) by appending the `-d` flag to the +above command.* -**:warning: Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged -`elastic` user. If you are starting the stack for the very first time, you MUST initialize a password for the [built-in -`kibana_system` user][builtin-users] to be able to start and access Kibana. Please read the section below attentively.** +Give Kibana about a minute to initialize, then access the Kibana web UI by opening in a web +browser and use the following (default) credentials to log in: + +* user: *elastic* +* password: *changeme* + +*:information_source: Upon the initial startup, the `elastic`, `logstash_internal` and `kibana_system` Elasticsearch +users are intialized with the values of the passwords defined in the [`.env`](.env) file (_"changeme"_ by default). The +first one is the [built-in superuser][builtin-users], the other two are used by Kibana and Logstash respectively to +communicate with Elasticsearch. This task is only performed during the _initial_ startup of the stack. To change users' +passwords _after_ they have been initialized, please refer to the instructions in the next section.* + +### Initial setup #### Setting up user authentication *:information_source: Refer to [Security settings in Elasticsearch][es-security] to disable authentication.* -The stack is pre-configured with the following **privileged** bootstrap user: +**:warning: Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged +`elastic` user.** -* user: *elastic* -* password: *changeme* +The _"changeme"_ password set by default for all aforementioned users is **unsecure**. For increased security, we will +reset the passwords of all aforementioned Elasticsearch users to random secrets. -For increased security, we will reset this bootstrap password, and generate a set of passwords to be used by -unprivileged [built-in users][builtin-users] within components of the Elastic stack. +1. Reset passwords for default users -1. Initialize passwords for built-in users + The commands below resets the passwords of the `elastic`, `logstash_internal` and `kibana_system` users. Take note + of them. - The commands below generate random passwords for the `elastic` and `kibana_system` users. Take note of them. + ```console + $ docker-compose exec elasticsearch bin/elasticsearch-reset-password --batch --user elastic + ``` ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user elastic + $ docker-compose exec elasticsearch bin/elasticsearch-reset-password --batch --user logstash_internal ``` ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system + $ docker-compose exec elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system ``` If the need for it arises (e.g. if you want to [collect monitoring information][ls-monitoring] through Beats and @@ -158,48 +171,40 @@ unprivileged [built-in users][builtin-users] within components of the Elastic st 1. Replace usernames and passwords in configuration files - Replace the password of the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) - with the password generated in the previous step. - - Replace the password of the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`) - with the password generated in the previous step. + Replace the password of the `elastic` user inside the `.env` file with the password generated in the previous step. + Its value isn't used by any core component, but [extensions](#how-to-enable-the-provided-extensions) use it to + connect to Elasticsearch. - *:information_source: Do not use the `logstash_system` user inside the Logstash **pipeline** file, it does not have - sufficient permissions to create indices. Follow the instructions at [Configuring Security in Logstash][ls-security] - to create a user with suitable roles.* + *:information_source: In case you don't plan on using any of the provided + [extensions](#how-to-enable-the-provided-extensions), or prefer to create your own roles and users to authenticate + these services, it is safe to remove the `ELASTIC_PASSWORD` entry from the `.env` file altogether after the stack + has been initialized.* - See also the [Configuration](#configuration) section below. + Replace the password of the `logstash_internal` user inside the `.env` file with the password generated in the + previous step. Its value is referenced inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`). -1. Unset the bootstrap password (_optional_) + Replace the password of the `kibana_system` user inside the `.env` file with the password generated in the previous + step. Its value is referenced inside the Kibana configuration file (`kibana/config/kibana.yml`). - Remove the `ELASTIC_PASSWORD` environment variable from the `elasticsearch` service inside the Compose file - (`docker-compose.yml`). It is only used to initialize the keystore during the initial startup of Elasticsearch, and - is ignored on subsequent runs. + See the [Configuration](#configuration) section below for more information about these configuration files. -1. Start Kibana and Logstash +1. Restart Logstash and Kibana to re-connect to Elasticsearch using the new passwords ```console - $ docker-compose up -d + $ docker-compose up -d logstash kibana ``` - The `-d` flag runs all services in the background (detached mode). - - On subsequent runs of the Elastic stack, it is sufficient to execute the above command in order to start all - components. - - *:information_source: Learn more about the security of the Elastic stack at [Secure the Elastic - Stack][sec-cluster].* +*:information_source: Learn more about the security of the Elastic stack at [Secure the Elastic Stack][sec-cluster].* #### Injecting data -Give Kibana about a minute to initialize, then access the Kibana web UI by opening in a web -browser and use the following credentials to log in: +Open the Kibana web UI by opening in a web browser and use the following credentials to log in: * user: *elastic* * password: *\* -Now that the stack is running, you can go ahead and inject some log entries. The shipped Logstash configuration allows -you to send content via TCP: +Now that the stack is fully configured, you can go ahead and inject some log entries. The shipped Logstash configuration +allows you to send content via TCP: ```console # Using BSD netcat (Debian, Ubuntu, MacOS system, ...) @@ -228,8 +233,9 @@ $ docker-compose down -v This repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major version (8.x). -To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If -you are upgrading an existing stack, please carefully read the note in the next section. +To use a different version of the core Elastic components, simply change the version number inside the [`.env`](.env) +file. If you are upgrading an existing stack, remember to rebuild all container images using the `docker-compose build` +command. **:warning: Always pay attention to the [official upgrade instructions][upgrade] for each individual component before performing a stack upgrade.** @@ -392,24 +398,6 @@ See the following Wiki pages: * [External applications](https://github.com/deviantony/docker-elk/wiki/External-applications) * [Popular integrations](https://github.com/deviantony/docker-elk/wiki/Popular-integrations) -### Swarm mode - -Experimental support for Docker [Swarm mode][swarm-mode] is provided in the form of a `docker-stack.yml` file, which can -be deployed in an existing Swarm cluster using the following command: - -```console -$ docker stack deploy -c docker-stack.yml elk -``` - -If all components get deployed without any error, the following command will show 3 running services: - -```console -$ docker stack services elk -``` - -*:information_source: To scale Elasticsearch in Swarm mode, configure seed hosts with the DNS name `tasks.elasticsearch` -instead of `elasticsearch`.* - [elk-stack]: https://www.elastic.co/what-is/elk-stack [xpack]: https://www.elastic.co/what-is/open-x-pack [paid-features]: https://www.elastic.co/subscriptions @@ -429,7 +417,6 @@ instead of `elasticsearch`.* [mac-filesharing]: https://docs.docker.com/desktop/mac/#file-sharing [builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html -[ls-security]: https://www.elastic.co/guide/en/logstash/current/ls-security.html [ls-monitoring]: https://www.elastic.co/guide/en/logstash/current/monitoring-with-metricbeat.html [sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-cluster.html @@ -445,5 +432,3 @@ instead of `elasticsearch`.* [ls-docker]: https://www.elastic.co/guide/en/logstash/current/docker-config.html [upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html - -[swarm-mode]: https://docs.docker.com/engine/swarm/ diff --git a/docker-compose.yml b/docker-compose.yml index 3411d1285c..8202cc85fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,33 @@ version: '3.2' services: + + # The 'setup' service runs a one-off script which initializes the + # 'logstash_internal' and 'kibana_system' users inside Elasticsearch with the + # values of the passwords defined in the '.env' file. + # + # This task is only performed during the *initial* startup of the stack. On all + # subsequent runs, the service simply returns immediately, without performing + # any modification to existing users. + setup: + build: + context: setup/ + args: + ELK_VERSION: ${ELK_VERSION} + volumes: + - setup:/state:Z + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} + LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-} + KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} + networks: + - elk + elasticsearch: build: context: elasticsearch/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} volumes: - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z - elasticsearch:/usr/share/elasticsearch/data:z @@ -14,7 +36,10 @@ services: - "9300:9300" environment: ES_JAVA_OPTS: -Xmx256m -Xms256m - ELASTIC_PASSWORD: 'changeme' + # Bootstrap password. + # Used to initialize the keystore during the initial startup of + # Elasticsearch. Ignored on subsequent runs. + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} # Use single node discovery in order to disable production mode and avoid bootstrap checks. # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html discovery.type: single-node @@ -25,7 +50,7 @@ services: build: context: logstash/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} volumes: - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z @@ -36,6 +61,7 @@ services: - "9600:9600" environment: LS_JAVA_OPTS: -Xmx256m -Xms256m + LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-} networks: - elk depends_on: @@ -45,11 +71,13 @@ services: build: context: kibana/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} volumes: - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z ports: - "5601:5601" + environment: + KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} networks: - elk depends_on: @@ -60,4 +88,5 @@ networks: driver: bridge volumes: + setup: elasticsearch: diff --git a/docker-stack.yml b/docker-stack.yml deleted file mode 100644 index f7eaad8f27..0000000000 --- a/docker-stack.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: '3.3' - -services: - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0 - ports: - - "9200:9200" - - "9300:9300" - configs: - - source: elastic_config - target: /usr/share/elasticsearch/config/elasticsearch.yml - environment: - ES_JAVA_OPTS: "-Xmx256m -Xms256m" - ELASTIC_PASSWORD: changeme - # Use single node discovery in order to disable production mode and avoid bootstrap checks. - # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html - discovery.type: single-node - # Force publishing on the 'elk' overlay. - network.publish_host: _eth0_ - networks: - - elk - deploy: - mode: replicated - replicas: 1 - - logstash: - image: docker.elastic.co/logstash/logstash:8.0.0 - ports: - - "5044:5044" - - "5000:5000" - - "9600:9600" - configs: - - source: logstash_config - target: /usr/share/logstash/config/logstash.yml - - source: logstash_pipeline - target: /usr/share/logstash/pipeline/logstash.conf - environment: - LS_JAVA_OPTS: "-Xmx256m -Xms256m" - networks: - - elk - deploy: - mode: replicated - replicas: 1 - - kibana: - image: docker.elastic.co/kibana/kibana:8.0.0 - ports: - - "5601:5601" - configs: - - source: kibana_config - target: /usr/share/kibana/config/kibana.yml - networks: - - elk - deploy: - mode: replicated - replicas: 1 - -configs: - - elastic_config: - file: ./elasticsearch/config/elasticsearch.yml - logstash_config: - file: ./logstash/config/logstash.yml - logstash_pipeline: - file: ./logstash/pipeline/logstash.conf - kibana_config: - file: ./kibana/config/kibana.yml - -networks: - elk: - driver: overlay diff --git a/elasticsearch/.dockerignore b/elasticsearch/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/elasticsearch/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/apm-server/.dockerignore b/extensions/apm-server/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/extensions/apm-server/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/apm-server/apm-server-compose.yml b/extensions/apm-server/apm-server-compose.yml index 991e553255..88bd73b596 100644 --- a/extensions/apm-server/apm-server-compose.yml +++ b/extensions/apm-server/apm-server-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/apm-server/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} command: # Disable strict permission checking on 'apm-server.yml' configuration file # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html @@ -14,6 +14,8 @@ services: - ./extensions/apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro,Z ports: - '8200:8200' + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} networks: - elk depends_on: diff --git a/extensions/apm-server/config/apm-server.yml b/extensions/apm-server/config/apm-server.yml index e78ea7d0b1..71e2ea9fb6 100644 --- a/extensions/apm-server/config/apm-server.yml +++ b/extensions/apm-server/config/apm-server.yml @@ -5,4 +5,4 @@ output: elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: 'changeme' + password: ${ELASTIC_PASSWORD} diff --git a/extensions/curator/.dockerignore b/extensions/curator/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/extensions/curator/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/enterprise-search/.dockerignore b/extensions/enterprise-search/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/extensions/enterprise-search/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index 49947c412a..40bb5923b3 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -22,7 +22,7 @@ kibana.host: http://localhost:5601 # Elasticsearch URL and credentials elasticsearch.host: http://elasticsearch:9200 elasticsearch.username: elastic -elasticsearch.password: 'changeme' +elasticsearch.password: ${ELASTIC_PASSWORD} # Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes. allow_es_settings_modification: true diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml index fcb12c14d1..90404ba092 100644 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ b/extensions/enterprise-search/enterprise-search-compose.yml @@ -5,12 +5,13 @@ services: build: context: extensions/enterprise-search/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} volumes: - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z environment: JAVA_OPTS: -Xmx2g -Xms2g ENT_SEARCH_DEFAULT_PASSWORD: 'changeme' + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} ports: - '3002:3002' networks: diff --git a/extensions/filebeat/.dockerignore b/extensions/filebeat/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/extensions/filebeat/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/filebeat/config/filebeat.yml b/extensions/filebeat/config/filebeat.yml index 18cf5f643c..dfbd0c3da3 100644 --- a/extensions/filebeat/config/filebeat.yml +++ b/extensions/filebeat/config/filebeat.yml @@ -20,7 +20,7 @@ processors: output.elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: 'changeme' + password: ${ELASTIC_PASSWORD} ## HTTP endpoint for health checking ## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml index 86dd11daa2..2e2ae1684f 100644 --- a/extensions/filebeat/filebeat-compose.yml +++ b/extensions/filebeat/filebeat-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/filebeat/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading # 'docker.sock' and the host's filesystem. user: root @@ -26,6 +26,8 @@ services: source: /var/run/docker.sock target: /var/run/docker.sock read_only: true + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} networks: - elk depends_on: diff --git a/extensions/logspout/.dockerignore b/extensions/logspout/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/extensions/logspout/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/metricbeat/.dockerignore b/extensions/metricbeat/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/extensions/metricbeat/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/metricbeat/config/metricbeat.yml b/extensions/metricbeat/config/metricbeat.yml index 3294035d82..f7c07ca6a6 100644 --- a/extensions/metricbeat/config/metricbeat.yml +++ b/extensions/metricbeat/config/metricbeat.yml @@ -34,7 +34,7 @@ processors: output.elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: 'changeme' + password: ${ELASTIC_PASSWORD} ## HTTP endpoint for health checking ## https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index 24770ed43a..8513958dd6 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/metricbeat/ args: - ELK_VERSION: $ELK_VERSION + ELK_VERSION: ${ELK_VERSION} # Run as 'root' instead of 'metricbeat' (uid 1000) to allow reading # 'docker.sock' and the host's filesystem. user: root @@ -37,6 +37,8 @@ services: source: /var/run/docker.sock target: /var/run/docker.sock read_only: true + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} networks: - elk depends_on: diff --git a/kibana/.dockerignore b/kibana/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/kibana/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 9025104d5b..07ab33e5cb 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -10,4 +10,4 @@ monitoring.ui.container.elasticsearch.enabled: true ## X-Pack security credentials # elasticsearch.username: kibana_system -elasticsearch.password: 'changeme' +elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD} diff --git a/logstash/.dockerignore b/logstash/.dockerignore new file mode 100644 index 0000000000..37eef9d513 --- /dev/null +++ b/logstash/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf index 40ca75775c..b61029b231 100644 --- a/logstash/pipeline/logstash.conf +++ b/logstash/pipeline/logstash.conf @@ -13,7 +13,7 @@ input { output { elasticsearch { hosts => "elasticsearch:9200" - user => "elastic" - password => "changeme" + user => "logstash_internal" + password => "${LOGSTASH_INTERNAL_PASSWORD}" } } diff --git a/setup/.dockerignore b/setup/.dockerignore new file mode 100644 index 0000000000..02f2244078 --- /dev/null +++ b/setup/.dockerignore @@ -0,0 +1,12 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store + +# Ignore Git files +.gitignore + +# Ignore setup state +state/ diff --git a/setup/.gitignore b/setup/.gitignore new file mode 100644 index 0000000000..a27475ad10 --- /dev/null +++ b/setup/.gitignore @@ -0,0 +1 @@ +/state/ diff --git a/setup/Dockerfile b/setup/Dockerfile new file mode 100644 index 0000000000..6c9305859f --- /dev/null +++ b/setup/Dockerfile @@ -0,0 +1,11 @@ +ARG ELK_VERSION + +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + +USER root +RUN mkdir /state && chown elasticsearch /state +USER elasticsearch:root + +COPY . / +ENTRYPOINT ["/entrypoint.sh"] diff --git a/setup/entrypoint.sh b/setup/entrypoint.sh new file mode 100755 index 0000000000..269bb4f605 --- /dev/null +++ b/setup/entrypoint.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/helpers.sh" + + +# -------------------------------------------------------- +# Users declarations + +declare -A users_passwords +users_passwords=( + [logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}" + [kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}" +) + +declare -A users_roles +users_roles=( + [logstash_internal]='logstash_writer' +) + +# -------------------------------------------------------- +# Roles declarations + +declare -A roles_files +roles_files=( + [logstash_writer]='logstash_writer.json' +) + +# -------------------------------------------------------- + + +echo "-------- $(date) --------" + +state_file="$(dirname ${BASH_SOURCE[0]})/state/.done" +if [[ -e "$state_file" ]]; then + log "State file exists at '${state_file}', skipping setup" + exit 0 +fi + +log 'Waiting for availability of Elasticsearch' +wait_for_elasticsearch +sublog 'Elasticsearch is running' + +for role in "${!roles_files[@]}"; do + log "Role '$role'" + + declare body_file + body_file="$(dirname "${BASH_SOURCE[0]}")/roles/${roles_files[$role]:-}" + if [[ ! -f "${body_file:-}" ]]; then + sublog "No role body found at '${body_file}', skipping" + continue + fi + + sublog 'Creating/updating' + ensure_role "$role" "$(<"${body_file}")" +done + +for user in "${!users_passwords[@]}"; do + log "User '$user'" + if [[ -z "${users_passwords[$user]:-}" ]]; then + sublog 'No password defined, skipping' + continue + fi + + declare -i user_exists=0 + user_exists="$(check_user_exists "$user")" + + if ((user_exists)); then + sublog 'User exists, setting password' + set_user_password "$user" "${users_passwords[$user]}" + else + if [[ -z "${users_roles[$user]:-}" ]]; then + err ' No role defined, skipping creation' + continue + fi + + sublog 'User does not exist, creating' + create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}" + fi +done + +mkdir -p "$(dirname "${state_file}")" +touch "$state_file" diff --git a/setup/helpers.sh b/setup/helpers.sh new file mode 100755 index 0000000000..24573725b8 --- /dev/null +++ b/setup/helpers.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +# Log a message. +function log { + echo "[+] $1" +} + +# Log a message at a sub-level. +function sublog { + echo " ⠿ $1" +} + +# Log an error. +function err { + echo "[x] $1" >&2 +} + +# Poll the 'elasticsearch' service until it responds with HTTP code 200. +function wait_for_elasticsearch { + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + # retry for max 300s (60*5s) + for _ in $(seq 1 60); do + output="$(curl "${args[@]}" || true)" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + break + fi + + sleep 5 + done + + if ((result)); then + echo -e "\n${output::-3}" + fi + + return $result +} + +# Verify that the given Elasticsearch user exists. +function check_user_exists { + local username=$1 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local -i exists=0 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then + result=0 + fi + if [[ "${output: -3}" -eq 200 ]]; then + exists=1 + fi + + if ((result)); then + echo -e "\n${output::-3}" + else + echo "$exists" + fi + + return $result +} + +# Set password of a given Elasticsearch user. +function set_user_password { + local username=$1 + local password=$2 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}/_password" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "{\"password\" : \"${password}\"}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} + +# Create the given Elasticsearch user. +function create_user { + local username=$1 + local password=$2 + local role=$3 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} + +# Ensure that the given Elasticsearch role is up-to-date, create it if required. +function ensure_role { + local name=$1 + local body=$2 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/role/${name}" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "$body" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} diff --git a/setup/roles/logstash_writer.json b/setup/roles/logstash_writer.json new file mode 100644 index 0000000000..6c857a8f7d --- /dev/null +++ b/setup/roles/logstash_writer.json @@ -0,0 +1,22 @@ +{ + "cluster": [ + "manage_index_templates", + "monitor", + "manage_ilm" + ], + "indices": [ + { + "names": [ + "logs-generic-default", + "logstash-*" + ], + "privileges": [ + "write", + "create", + "create_index", + "manage", + "manage_ilm" + ] + } + ] +} From 0a9c3ebd2cf943c416cc62928a5687c4e475d750 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 28 Feb 2022 09:02:44 +0100 Subject: [PATCH 39/55] doc: Update "Philosophy" section to reflect latest changes We do use a small amount of custom setup automation now. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0747dba42..2854711e8e 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ Other available stack variants: We aim at providing the simplest possible entry into the Elastic stack for anybody who feels like experimenting with this powerful combo of technologies. This project's default configuration is purposely minimal and unopinionated. It -does not rely on any external dependency or custom automation to get things up and running. +does not rely on any external dependency, and uses as little custom automation as necessary to get things up and +running. Instead, we believe in good documentation so that you can use this repository as a template, tweak it, and make it _your own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of project that builds upon this idea. From eb1c6fa27e730a40b171d541d5c991ab243522c7 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 28 Feb 2022 09:10:30 +0100 Subject: [PATCH 40/55] chore: Rename ELK_VERSION to ELASTIC_VERSION More appropriate name considering that extensions, which aren't part of the "ELK" stack, also rely on this variable. --- .env | 2 +- .github/workflows/update.yml | 2 +- docker-compose.yml | 8 ++++---- elasticsearch/Dockerfile | 4 ++-- extensions/apm-server/Dockerfile | 4 ++-- extensions/apm-server/apm-server-compose.yml | 2 +- extensions/enterprise-search/Dockerfile | 4 ++-- .../enterprise-search/enterprise-search-compose.yml | 2 +- extensions/filebeat/Dockerfile | 4 ++-- extensions/filebeat/filebeat-compose.yml | 2 +- extensions/metricbeat/Dockerfile | 4 ++-- extensions/metricbeat/metricbeat-compose.yml | 2 +- kibana/Dockerfile | 4 ++-- logstash/Dockerfile | 4 ++-- setup/Dockerfile | 4 ++-- 15 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.env b/.env index b4197807bd..41d43a9c74 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELK_VERSION=8.0.0 +ELASTIC_VERSION=8.0.0 ## Passwords for stack users # diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index e0a0f03dab..d9d8cfad10 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -65,7 +65,7 @@ jobs: if: steps.get-latest-release.outputs.result run: | source .env - cur_ver="$ELK_VERSION" + cur_ver="$ELASTIC_VERSION" new_ver=${{ fromJson(steps.get-latest-release.outputs.result).version }} # Escape dot characters so sed interprets them as literal dots diff --git a/docker-compose.yml b/docker-compose.yml index 8202cc85fc..b9e02c6ff1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: build: context: setup/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - setup:/state:Z environment: @@ -27,7 +27,7 @@ services: build: context: elasticsearch/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z - elasticsearch:/usr/share/elasticsearch/data:z @@ -50,7 +50,7 @@ services: build: context: logstash/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z @@ -71,7 +71,7 @@ services: build: context: kibana/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z ports: diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 39285445dd..22528c6d7b 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} # Add your elasticsearch plugins setup here # Example: RUN elasticsearch-plugin install analysis-icu diff --git a/extensions/apm-server/Dockerfile b/extensions/apm-server/Dockerfile index 2c611df90a..9f03f56229 100644 --- a/extensions/apm-server/Dockerfile +++ b/extensions/apm-server/Dockerfile @@ -1,3 +1,3 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION -FROM docker.elastic.co/apm/apm-server:${ELK_VERSION} +FROM docker.elastic.co/apm/apm-server:${ELASTIC_VERSION} diff --git a/extensions/apm-server/apm-server-compose.yml b/extensions/apm-server/apm-server-compose.yml index 88bd73b596..b951d4b9c5 100644 --- a/extensions/apm-server/apm-server-compose.yml +++ b/extensions/apm-server/apm-server-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/apm-server/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} command: # Disable strict permission checking on 'apm-server.yml' configuration file # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html diff --git a/extensions/enterprise-search/Dockerfile b/extensions/enterprise-search/Dockerfile index 7edd72094e..4f0752e55a 100644 --- a/extensions/enterprise-search/Dockerfile +++ b/extensions/enterprise-search/Dockerfile @@ -1,4 +1,4 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/enterprise-search/enterprise-search:${ELK_VERSION} +FROM docker.elastic.co/enterprise-search/enterprise-search:${ELASTIC_VERSION} diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml index 90404ba092..5b9e9fe61a 100644 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ b/extensions/enterprise-search/enterprise-search-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/enterprise-search/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z environment: diff --git a/extensions/filebeat/Dockerfile b/extensions/filebeat/Dockerfile index b70aa10308..b8dd5f3f5a 100644 --- a/extensions/filebeat/Dockerfile +++ b/extensions/filebeat/Dockerfile @@ -1,3 +1,3 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/filebeat:${ELK_VERSION} +FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION} diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml index 2e2ae1684f..020336e875 100644 --- a/extensions/filebeat/filebeat-compose.yml +++ b/extensions/filebeat/filebeat-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/filebeat/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading # 'docker.sock' and the host's filesystem. user: root diff --git a/extensions/metricbeat/Dockerfile b/extensions/metricbeat/Dockerfile index e55f62515d..6d05bf55f2 100644 --- a/extensions/metricbeat/Dockerfile +++ b/extensions/metricbeat/Dockerfile @@ -1,3 +1,3 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/metricbeat:${ELK_VERSION} +FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION} diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index 8513958dd6..7777887e25 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -5,7 +5,7 @@ services: build: context: extensions/metricbeat/ args: - ELK_VERSION: ${ELK_VERSION} + ELASTIC_VERSION: ${ELASTIC_VERSION} # Run as 'root' instead of 'metricbeat' (uid 1000) to allow reading # 'docker.sock' and the host's filesystem. user: root diff --git a/kibana/Dockerfile b/kibana/Dockerfile index 2fb3659b35..9a075bedb9 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} +FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION} # Add your kibana plugins setup here # Example: RUN kibana-plugin install diff --git a/logstash/Dockerfile b/logstash/Dockerfile index 6a444e7b99..bde5808d98 100644 --- a/logstash/Dockerfile +++ b/logstash/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} +FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION} # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json diff --git a/setup/Dockerfile b/setup/Dockerfile index 6c9305859f..21f16e8c41 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} USER root RUN mkdir /state && chown elasticsearch /state From a204596be77e43b3f2d4f715362966c761a23297 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 28 Feb 2022 09:46:27 +0100 Subject: [PATCH 41/55] fix: Explicitly make entrypoint of 'setup' service executable The existing executable permission might not be preserved when users download docker-elk as a Zip file. Fixes #676 --- setup/Dockerfile | 10 ++++++++-- setup/helpers.sh | 0 2 files changed, 8 insertions(+), 2 deletions(-) mode change 100755 => 100644 setup/helpers.sh diff --git a/setup/Dockerfile b/setup/Dockerfile index 21f16e8c41..0b2ef9094f 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -4,8 +4,14 @@ ARG ELASTIC_VERSION FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} USER root -RUN mkdir /state && chown elasticsearch /state -USER elasticsearch:root COPY . / + +RUN set -eux; \ + mkdir /state; \ + chown elasticsearch /state; \ + chmod +x /entrypoint.sh + +USER elasticsearch:root + ENTRYPOINT ["/entrypoint.sh"] diff --git a/setup/helpers.sh b/setup/helpers.sh old mode 100755 new mode 100644 From 924a1e5987e31d0a8800ab091c1b91b0c216a8c0 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 28 Feb 2022 09:50:52 +0100 Subject: [PATCH 42/55] chore: Update Compose file version to 3.7 Allows enabling the 'init' option in the 'setup' service. Requires Docker 18.06.0 and Compose 1.22.0. --- README.md | 4 ++-- docker-compose.yml | 3 ++- extensions/apm-server/apm-server-compose.yml | 2 +- extensions/curator/curator-compose.yml | 2 +- extensions/enterprise-search/enterprise-search-compose.yml | 2 +- extensions/filebeat/filebeat-compose.yml | 2 +- extensions/logspout/logspout-compose.yml | 2 +- extensions/metricbeat/metricbeat-compose.yml | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2854711e8e..ffe4c9bd8c 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p ### Host setup -* [Docker Engine](https://docs.docker.com/install/) version **17.05** or newer -* [Docker Compose](https://docs.docker.com/compose/install/) version **1.20.0** or newer +* [Docker Engine](https://docs.docker.com/install/) version **18.06.0** or newer +* [Docker Compose](https://docs.docker.com/compose/install/) version **1.22.0** or newer * 1.5 GB of RAM *:information_source: Especially on Linux, make sure your user has the [required permissions][linux-postinstall] to diff --git a/docker-compose.yml b/docker-compose.yml index b9e02c6ff1..1f85ef6141 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: @@ -14,6 +14,7 @@ services: context: setup/ args: ELASTIC_VERSION: ${ELASTIC_VERSION} + init: true volumes: - setup:/state:Z environment: diff --git a/extensions/apm-server/apm-server-compose.yml b/extensions/apm-server/apm-server-compose.yml index b951d4b9c5..9d5d2b10f1 100644 --- a/extensions/apm-server/apm-server-compose.yml +++ b/extensions/apm-server/apm-server-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: apm-server: diff --git a/extensions/curator/curator-compose.yml b/extensions/curator/curator-compose.yml index 60786a6628..488cd8036c 100644 --- a/extensions/curator/curator-compose.yml +++ b/extensions/curator/curator-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: curator: diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml index 5b9e9fe61a..f6df917c66 100644 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ b/extensions/enterprise-search/enterprise-search-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: enterprise-search: diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml index 020336e875..04ff104048 100644 --- a/extensions/filebeat/filebeat-compose.yml +++ b/extensions/filebeat/filebeat-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: filebeat: diff --git a/extensions/logspout/logspout-compose.yml b/extensions/logspout/logspout-compose.yml index bb60bef018..a603c3fed7 100644 --- a/extensions/logspout/logspout-compose.yml +++ b/extensions/logspout/logspout-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: logspout: diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index 7777887e25..da62833ef0 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: metricbeat: From 9a52c6cf5f2d6ae642bb2bf6ce5941f07d7ad3a4 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 28 Feb 2022 12:03:18 +0100 Subject: [PATCH 43/55] chore: Update animated demo for v8.0.0 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffe4c9bd8c..42b9ea3ef7 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker It gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and the visualization power of Kibana. -![Animated demo](https://user-images.githubusercontent.com/3299086/140641708-cea70d17-cc04-459f-89d9-3fcb5c58bc35.gif) +![Animated demo](https://user-images.githubusercontent.com/3299086/155972072-0c89d6db-707a-47a1-818b-5f976565f95a.gif) *:information_source: The Docker images backing this stack include [X-Pack][xpack] with [paid features][paid-features] enabled by default (see [How to disable paid features](#how-to-disable-paid-features) to disable them). **The [trial From 159c934ad28a62a4022beab23ce4fb98d5c5c98f Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Mon, 28 Feb 2022 15:33:50 +0100 Subject: [PATCH 44/55] doc: Bump minimum Compose version requirement to 1.26.0 Prior to this version, surrounding quotes (single and double) were preserved in values coming from the '.env' file, which is counter intuitive and goes against the recommended practice of quoting values containing special characters, such as passwords. Closes #677 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42b9ea3ef7..d52faca108 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p ### Host setup * [Docker Engine](https://docs.docker.com/install/) version **18.06.0** or newer -* [Docker Compose](https://docs.docker.com/compose/install/) version **1.22.0** or newer +* [Docker Compose](https://docs.docker.com/compose/install/) version **1.26.0** or newer * 1.5 GB of RAM *:information_source: Especially on Linux, make sure your user has the [required permissions][linux-postinstall] to From 02dabf6fb1b8718cc16add369eaed380575d837f Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Wed, 2 Mar 2022 16:34:48 +0100 Subject: [PATCH 45/55] fix(logstash): Allow managing "logstash" index When data streams are disabled in Logstash's "elasticsearch" output ('data_stream => false'), Logstash falls back to managing "logstash-*" indices and creates an index template with ILM enabled. In this process, a write index for the alias "logstash" is created. This step requires the "manage" privilege on the "logstash" alias. Fixes #679 --- setup/roles/logstash_writer.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/setup/roles/logstash_writer.json b/setup/roles/logstash_writer.json index 6c857a8f7d..7cd38fc985 100644 --- a/setup/roles/logstash_writer.json +++ b/setup/roles/logstash_writer.json @@ -17,6 +17,15 @@ "manage", "manage_ilm" ] + }, + { + "names": [ + "logstash" + ], + "privileges": [ + "write", + "manage" + ] } ] } From 82b1e653b09619396ebfe84624462151fbfc9308 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Wed, 2 Mar 2022 16:58:52 +0100 Subject: [PATCH 46/55] fix(logstash): Adjust role for ECS compatibility 'ecs_compatibility' is now on by default, therefore Logstash indices are created with the naming pattern "ecs-logstash-*" when data streams are disabled ('data_stream => false'). --- setup/roles/logstash_writer.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/roles/logstash_writer.json b/setup/roles/logstash_writer.json index 7cd38fc985..b43861fed9 100644 --- a/setup/roles/logstash_writer.json +++ b/setup/roles/logstash_writer.json @@ -8,7 +8,8 @@ { "names": [ "logs-generic-default", - "logstash-*" + "logstash-*", + "ecs-logstash-*" ], "privileges": [ "write", @@ -20,7 +21,8 @@ }, { "names": [ - "logstash" + "logstash", + "ecs-logstash" ], "privileges": [ "write", From 897720fa91300f6090049e091625cd265b8c44ea Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sun, 6 Mar 2022 10:28:48 +0100 Subject: [PATCH 47/55] ci: Pin Compose version to v2.2.3 v2.3.0 introduced a regression which causes the test CI workflow to fail: https://github.com/docker/compose/issues/9234 --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5fd2bfd2f..b5b0c95eff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: ##################################################### - name: Prepare environment + env: + COMPOSE_VERSION: v2.2.3 run: | # Enable support for Compose V2 @@ -32,7 +34,7 @@ jobs: # https://docs.docker.com/compose/cli-command/#install-on-linux mkdir -p ~/.docker/cli-plugins - curl -sL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose + curl -sL "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose docker compose version From 21aa0ce9afea8782203f368ec5e677d9184ed3e7 Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Sun, 6 Mar 2022 10:00:13 +0000 Subject: [PATCH 48/55] Update to v8.0.1 --- .env | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 41d43a9c74..25ef1762b0 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.0.0 +ELASTIC_VERSION=8.0.1 ## Passwords for stack users # diff --git a/README.md b/README.md index d52faca108..615129bfe0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -143,7 +143,7 @@ passwords _after_ they have been initialized, please refer to the instructions i *:information_source: Refer to [Security settings in Elasticsearch][es-security] to disable authentication.* -**:warning: Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged +**:warning: Starting with Elastic v8.0.1, it is no longer possible to run Kibana using the bootstraped privileged `elastic` user.** The _"changeme"_ password set by default for all aforementioned users is **unsecure**. For increased security, we will From 488a88b4ca6cd6f2562ec53c2da7cfcfd0b15704 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sun, 6 Mar 2022 11:02:29 +0100 Subject: [PATCH 49/55] doc: Revert in-paragraph string replacement by update bot --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 615129bfe0..0c534f4db9 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ passwords _after_ they have been initialized, please refer to the instructions i *:information_source: Refer to [Security settings in Elasticsearch][es-security] to disable authentication.* -**:warning: Starting with Elastic v8.0.1, it is no longer possible to run Kibana using the bootstraped privileged +**:warning: Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged `elastic` user.** The _"changeme"_ password set by default for all aforementioned users is **unsecure**. For increased security, we will From 4fe6f6833961369e98c1a57c81222053584210ed Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sun, 6 Mar 2022 11:07:29 +0100 Subject: [PATCH 50/55] ci: Update list of auto-mergeable branch names --- .github/workflows/update-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-merge.yml b/.github/workflows/update-merge.yml index b5fbf741d0..08457b0f75 100644 --- a/.github/workflows/update-merge.yml +++ b/.github/workflows/update-merge.yml @@ -7,7 +7,7 @@ on: - completed branches: - update/main - - update/release-6.x + - update/release-7.x jobs: From 12dc0a4b74b2e983af36e4e2b04cead202427e85 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Wed, 9 Mar 2022 01:57:19 +0100 Subject: [PATCH 51/55] Revert "ci: Pin Compose version to v2.2.3" This reverts commit 897720fa91300f6090049e091625cd265b8c44ea. A new version of Compose was released, with a fix for the issue we were encountering. --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5b0c95eff..b5fd2bfd2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,8 +24,6 @@ jobs: ##################################################### - name: Prepare environment - env: - COMPOSE_VERSION: v2.2.3 run: | # Enable support for Compose V2 @@ -34,7 +32,7 @@ jobs: # https://docs.docker.com/compose/cli-command/#install-on-linux mkdir -p ~/.docker/cli-plugins - curl -sL "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose + curl -sL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose docker compose version From 1f26484b61fe7df3358bacfde7ede0489d777364 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 12 Mar 2022 17:25:20 +0100 Subject: [PATCH 52/55] doc: Mention support for Compose V2 Closes #684 --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0c534f4db9..369d559542 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p ### Host setup -* [Docker Engine](https://docs.docker.com/install/) version **18.06.0** or newer -* [Docker Compose](https://docs.docker.com/compose/install/) version **1.26.0** or newer +* [Docker Engine][docker-install] version **18.06.0** or newer +* [Docker Compose][compose-install] version **1.26.0** or newer (including [Compose V2][compose-v2]) * 1.5 GB of RAM *:information_source: Especially on Linux, make sure your user has the [required permissions][linux-postinstall] to @@ -409,7 +409,10 @@ See the following Wiki pages: [elastdocker]: https://github.com/sherifabdlnaby/elastdocker -[linux-postinstall]: https://docs.docker.com/install/linux/linux-postinstall/ +[docker-install]: https://docs.docker.com/get-docker/ +[compose-install]: https://docs.docker.com/compose/install/ +[compose-v2]: https://docs.docker.com/compose/cli-command/ +[linux-postinstall]: https://docs.docker.com/engine/install/linux-postinstall/ [booststap-checks]: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html [es-sys-config]: https://www.elastic.co/guide/en/elasticsearch/reference/current/system-config.html From 0f5dcf1d70519591f2ec0a51f7d4044d0476dddd Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Sun, 13 Mar 2022 08:10:50 +0000 Subject: [PATCH 53/55] Update to v8.1.0 --- .env | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 25ef1762b0..7b4612fcb4 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.0.1 +ELASTIC_VERSION=8.1.0 ## Passwords for stack users # diff --git a/README.md b/README.md index 369d559542..b4d1ca50ed 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.1.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From 2ec05d65b25548f3528cf7a9425e5af128820726 Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Sun, 27 Mar 2022 00:22:47 +0000 Subject: [PATCH 54/55] Update to v8.1.1 --- .env | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 7b4612fcb4..6a4f51e33a 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.1.0 +ELASTIC_VERSION=8.1.1 ## Passwords for stack users # diff --git a/README.md b/README.md index b4d1ca50ed..e1c3619bef 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.1.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.1.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From d9d43c2a400fcf0643ff2565b1c58b18acb03c72 Mon Sep 17 00:00:00 2001 From: "docker-elk-updater[bot]" <93581810+docker-elk-updater[bot]@users.noreply.github.com> Date: Sun, 3 Apr 2022 00:21:55 +0000 Subject: [PATCH 55/55] Update to v8.1.2 --- .env | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 6a4f51e33a..0831c55268 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.1.1 +ELASTIC_VERSION=8.1.2 ## Passwords for stack users # diff --git a/README.md b/README.md index e1c3619bef..2b4b1209ae 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.1.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.1.2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)