Skip to content

Commit

Permalink
Merge branch 'deviantony:main' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@waitesgithub
waitesgithub authored Apr 18, 2022
2 parents 48a219e + d9d43c2 commit 3e6970e
Show file tree
Hide file tree
Showing 55 changed files with 1,098 additions and 533 deletions.
2 changes: 0 additions & 2 deletions .dockerignore
View file

This file was deleted.

23 changes: 22 additions & 1 deletion .env
View file
Original file line number Diff line number Diff line change
@@ -1 +1,22 @@
ELK_VERSION=7.14.1
ELASTIC_VERSION=8.1.2

## Passwords for stack users
#

# User 'elastic' (built-in)
#
# Superuser role, full access to cluster management and data indices.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
ELASTIC_PASSWORD='changeme'

# User 'logstash_internal' (custom)
#
# The user Logstash uses to connect and send data to Elasticsearch.
# https://www.elastic.co/guide/en/logstash/current/ls-security.html
LOGSTASH_INTERNAL_PASSWORD='changeme'

# User 'kibana_system' (built-in)
#
# The user Kibana uses to connect and communicate with Elasticsearch.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
KIBANA_SYSTEM_PASSWORD='changeme'
206 changes: 71 additions & 135 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ on:

jobs:

test-compose:
name: 'Test suite: Compose'
test:
name: Test suite
# List of supported runners:
# https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources
runs-on: ubuntu-latest
Expand All @@ -26,16 +26,26 @@ jobs:
- name: Prepare environment
run: |
# Install Linux packages
# Enable support for Compose V2
#
# List of packages pre-installed in the runner:
# https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-software
# Instructions:
# https://docs.docker.com/compose/cli-command/#install-on-linux
sudo apt install -y expect
mkdir -p ~/.docker/cli-plugins
curl -sL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64" -o ~/.docker/cli-plugins/docker-compose
chmod +x ~/.docker/cli-plugins/docker-compose
docker compose version
# Pre-build container images
docker-compose build
docker compose \
-f docker-compose.yml \
-f extensions/logspout/logspout-compose.yml \
-f extensions/enterprise-search/enterprise-search-compose.yml \
-f extensions/apm-server/apm-server-compose.yml \
-f extensions/metricbeat/metricbeat-compose.yml \
-f extensions/filebeat/filebeat-compose.yml \
build
########################################################
# #
Expand All @@ -46,24 +56,9 @@ jobs:
- name: Set password of every built-in user to 'testpasswd'
run: |
# Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files
sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml
sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf
sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana_system/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml
sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml
sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml
# Run Elasticsearch and wait for its availability
docker-compose up -d elasticsearch
source .github/workflows/scripts/lib/testing.sh
poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme'
# Set passwords
.github/workflows/scripts/elasticsearch-setup-passwords.exp
sed -i -e 's/\(ELASTIC_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
-e 's/\(LOGSTASH_INTERNAL_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
-e 's/\(KIBANA_SYSTEM_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' .env
##########################################################
# #
Expand All @@ -72,7 +67,7 @@ jobs:
##########################################################

- name: Run the stack
run: docker-compose up -d
run: docker compose up -d

- name: Execute core test suite
run: .github/workflows/scripts/run-tests-core.sh
Expand All @@ -82,10 +77,11 @@ jobs:
# https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#job-status-check-functions
if: always()
run: |
docker-compose ps
docker-compose logs elasticsearch
docker-compose logs logstash
docker-compose logs kibana
docker compose ps
docker compose logs setup
docker compose logs elasticsearch
docker compose logs logstash
docker compose logs kibana
##############################
# #
Expand All @@ -104,26 +100,29 @@ jobs:
sed -i '$ a input { udp { port => 5000 codec => json } }' logstash/pipeline/logstash.conf
# Restart Logstash for changes to take effect
docker compose restart logstash
# Run Logspout and execute tests.
# Logstash will be restarted as a result of building the Logspout
# image, so changes above will automatically take effect.
docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up -d logspout
docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up -d logspout
.github/workflows/scripts/run-tests-logspout.sh
# Revert changes to Logstash configuration
sed -i '/input { udp { port => 5000 codec => json } }/d' logstash/pipeline/logstash.conf
docker compose restart logstash
- name: 'debug: Display state and logs (Logspout)'
if: always()
run: |
docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps
docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout
docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps
docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout
# next steps don't need Logspout
docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml stop logspout
docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml stop logspout
# next steps don't need Logstash
docker-compose stop logstash
docker compose stop logstash
#
# Enterprise Search
Expand All @@ -138,59 +137,78 @@ jobs:
# Restart Elasticsearch for changes to take effect
docker-compose restart elasticsearch
docker compose restart elasticsearch
# Run Enterprise Search and execute tests
docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -d enterprise-search
sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -d enterprise-search
.github/workflows/scripts/run-tests-enterprise-search.sh
# Revert changes to Elasticsearch configuration
sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
docker-compose restart elasticsearch
docker compose restart elasticsearch
- name: 'debug: Display state and logs (Enterprise Search)'
if: always()
run: |
docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml ps
docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search
docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml ps
docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search
# next steps don't need Enterprise Search
docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml stop enterprise-search
docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml stop enterprise-search
#
# APM Server
#

- name: Execute APM Server test suite
run: |
docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up -d apm-server
docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up -d apm-server
.github/workflows/scripts/run-tests-apm-server.sh
- name: 'debug: Display state and logs (APM Server)'
if: always()
run: |
docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml ps
docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml logs apm-server
docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml ps
docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml logs apm-server
# next steps don't need APM Server
docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml stop apm-server
docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml stop apm-server
#
# Metricbeat
#

- name: Execute Metricbeat test suite
run: |
docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up -d metricbeat
docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up -d metricbeat
.github/workflows/scripts/run-tests-metricbeat.sh
- name: 'debug: Display state and logs (Metricbeat)'
if: always()
run: |
docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps
docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat
docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps
docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat
# next steps don't need Metricbeat
docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat
docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat
#
# Filebeat
#

- name: Execute Filebeat test suite
run: |
docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up -d filebeat
.github/workflows/scripts/run-tests-filebeat.sh
- name: 'debug: Display state and logs (Filebeat)'
if: always()
run: |
docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml ps
docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml logs filebeat
# next steps don't need Filebeat
docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml stop filebeat
##############
# #
Expand All @@ -201,93 +219,11 @@ jobs:
- name: Terminate all components
if: always()
run: >-
docker-compose
docker compose
-f docker-compose.yml
-f extensions/logspout/logspout-compose.yml
-f extensions/enterprise-search/enterprise-search-compose.yml
-f extensions/apm-server/apm-server-compose.yml
-f extensions/metricbeat/metricbeat-compose.yml
-f extensions/filebeat/filebeat-compose.yml
down -v
test-swarm:
name: 'Test suite: Swarm'
runs-on: ubuntu-latest

env:
MODE: swarm

steps:
- uses: actions/checkout@v2

#####################################################
# #
# Install all dependencies required by test suites. #
# #
#####################################################

- name: Prepare environment
run: |
# Install Linux packages
sudo apt install -y expect
# Enable Swarm mode
docker swarm init
########################################################
# #
# Ensure §"Initial setup" of the README remains valid. #
# #
########################################################

- name: Set password of every built-in user to 'testpasswd'
run: |
# Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files
sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml
sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf
sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana_system/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml
# Run Elasticsearch and wait for its availability
docker stack deploy -c ./docker-stack.yml elk
docker service scale elk_logstash=0 elk_kibana=0
source .github/workflows/scripts/lib/testing.sh
poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme'
# Set passwords
.github/workflows/scripts/elasticsearch-setup-passwords.exp swarm
##########################################################
# #
# Test core components: Elasticsearch, Logstash, Kibana. #
# #
##########################################################

- name: Run the stack
run: docker service scale elk_logstash=1 elk_kibana=1

- name: Execute core test suite
run: .github/workflows/scripts/run-tests-core.sh swarm

- name: 'debug: Display state and logs (core)'
if: always()
run: |
docker stack services elk
docker service logs elk_elasticsearch
docker service logs elk_kibana
docker service logs elk_logstash
##############
# #
# Tear down. #
# #
##############

- name: Terminate all components
if: always()
run: docker stack rm elk
28 changes: 0 additions & 28 deletions .github/workflows/scripts/elasticsearch-setup-passwords.exp
View file

This file was deleted.

Loading

0 comments on commit 3e6970e

Please sign in to comment.