From c6f4afd7ca8a6e95e005b3f8b84b34ac07f5257b Mon Sep 17 00:00:00 2001 From: Saikumar Date: Tue, 28 Jan 2025 11:48:37 +0530 Subject: [PATCH] Updates to Support Community Versions --- pmm_qa/mlaunch_modb_setup.sh | 100 ++++++++++++++++++ pmm_qa/mlaunch_modb_setup.yml | 85 +++++++++++++++ ...launch_setup.sh => mlaunch_psmdb_setup.sh} | 17 ++- ...unch_setup.yml => mlaunch_psmdb_setup.yml} | 4 +- pmm_qa/pmm-framework.py | 44 +++++++- 5 files changed, 241 insertions(+), 9 deletions(-) create mode 100755 pmm_qa/mlaunch_modb_setup.sh create mode 100644 pmm_qa/mlaunch_modb_setup.yml rename pmm_qa/{mlaunch_setup.sh => mlaunch_psmdb_setup.sh} (92%) rename pmm_qa/{mlaunch_setup.yml => mlaunch_psmdb_setup.yml} (94%) diff --git a/pmm_qa/mlaunch_modb_setup.sh b/pmm_qa/mlaunch_modb_setup.sh new file mode 100755 index 0000000..3652390 --- /dev/null +++ b/pmm_qa/mlaunch_modb_setup.sh @@ -0,0 +1,100 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + shift +done + +if [ -z "$mongodb_version" ]; then + export mongodb_version=7.0 +fi + +if [ -z "$mongdb_setup" ]; then + export mongdb_setup=pss +fi + +if [ -z "$metrics_mode" ]; then + export metrics_mode=push +fi + +# Mongo user credtials for the replicat set cluster +export user="dba" +export pwd="test1234" + +# Install the dependencies +source ~/.bash_profile || true; +apt-get update +apt-get -y install wget curl jq git gnupg2 lsb-release +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl +sleep 10 + +#wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh +#chmod +x mongo_startup.sh +export SERVICE_RANDOM_NUMBER=$(echo $((1 + $RANDOM % 9999))) + +## Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +modb_tarball=$(curl -s "https://www.mongodb.com/try/download/community" | \ + grep -oP "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2004-${mongodb_version//./\\.}\.\d+\.tgz" | \ + sort -V | tail -n 1) + +echo "Downloading ${mongodb_version} from ${modb_tarball}..." +wget -O percona_server_mongodb.tar.gz ${modb_tarball} +tar -xvf percona_server_mongodb.tar.gz + +export extracted_folder_name=$(ls | grep mongodb-linux) +echo "Extracted folder name ${extracted_folder_name}" +mv ${extracted_folder_name} modb_${mongodb_version} +rm percona_server_mongodb.tar.gz* + +# For mongodb dependency in Debian +wget http://http.us.debian.org/debian/pool/main/o/openldap/libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb +apt install -y ./libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb + +if [ "$mongodb_setup" == "pss" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./modb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 3 + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +if [ "$mongodb_setup" == "psa" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./modb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 2 --arbiter + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +if [ "$mongodb_setup" == "sharded" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./modb_${mongodb_version}/bin" --replicaset --sharded rs1 rs2 --config 3 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --environment=mongos_shraded_node mongos_shraded_node_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27017 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27024 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27025 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27026 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27018 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27019 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27020 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27021 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27022 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27023 + sleep 20 +fi diff --git a/pmm_qa/mlaunch_modb_setup.yml b/pmm_qa/mlaunch_modb_setup.yml new file mode 100644 index 0000000..bea4d69 --- /dev/null +++ b/pmm_qa/mlaunch_modb_setup.yml @@ -0,0 +1,85 @@ +--- + +- hosts: all + become: true + become_method: sudo + vars: + modb_version: "{{ lookup('vars', 'extra_modb_version', default=lookup('env','MODB_VERSION') | default('4.4', true) ) }}" + modb_tarball: "{{ lookup('vars', 'extra_modb_tarball', default=lookup('env','MODB_TARBALL') | default('', true) ) }}" + modb_setup: "{{ lookup('vars', 'extra_modb_setup', default=lookup('env','MODB_SETUP') | default('pss', true) ) }}" + modb_container: "{{ lookup('vars', 'extra_modb_container', default=lookup('env','MODB_CONTAINER') | default('MODB', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ modb_container }}" | grep -q . && docker stop {{ modb_container }} && docker rm -fv {{ modb_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for modb + shell: > + docker run -d --name={{ modb_container }} + -p 27017:27017 + phusion/baseimage:focal-1.2.0 + + - name: Copy all required Artifacts to the docker modb_container + shell: "{{ item }}" + with_items: + - docker cp ./mlaunch_modb_setup.sh {{ modb_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ modb_container }}:/ + - docker exec {{ modb_container }} apt-get update + - docker exec {{ modb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq python3 pip + + - name: Install required software's to the docker modb_container + shell: "{{ item }}" + with_items: + - docker exec {{ modb_container }} python3 -m pip install --upgrade pip + - docker exec {{ modb_container }} pip3 install 'mtools[all]' + + - name: Install pmm2-client on the modb_container + shell: "{{ item }}" + with_items: + - docker network connect pmm-qa {{ modb_container }} + - docker exec {{ modb_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Setup modb for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ modb_container }} bash -x ./mlaunch_modb_setup.sh --mongodb_version {{ modb_version }} --mongodb_setup {{ modb_setup }} > setup_modb_{{ modb_version }}_{{ modb_setup }}.log + + - name: Setup Load Running Docker Container + shell: "{{ item }}" + with_items: + - rm -rf ~/modb_{{ modb_version }} || true; mkdir -p ~/modb_{{ modb_version }} + - wget -P ~/modb_{{ modb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/pmm3-mongo-mlaunch/pmm_qa/Dockerfile" + - wget -P ~/modb_{{ modb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/pmm3-mongo-mlaunch/pmm_qa/mongodb_query.php" + - docker build --tag php-db ~/modb_{{ modb_version }}/ > ~/docker-build_mongodb_load_{{ modb_version }}_{{ modb_setup }}.log || true + - docker rm mongodb_load_{{ modb_version }}_{{ modb_setup }} || true + - docker run --rm --name mongodb_load_{{ modb_version }}_{{ modb_setup }} --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db composer require mongodb/mongodb || true + + - name: Run load on Replica Set Master(PSS) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log + when: modb_setup == "pss" + + - name: Run load on Replica Set Master(PSA) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log + when: modb_setup == "psa" + + - name: Run load on Sharded Clusters Master + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log + when: modb_setup == "sharded" \ No newline at end of file diff --git a/pmm_qa/mlaunch_setup.sh b/pmm_qa/mlaunch_psmdb_setup.sh similarity index 92% rename from pmm_qa/mlaunch_setup.sh rename to pmm_qa/mlaunch_psmdb_setup.sh index 119ad8a..9c73756 100755 --- a/pmm_qa/mlaunch_setup.sh +++ b/pmm_qa/mlaunch_psmdb_setup.sh @@ -9,7 +9,7 @@ while [ $# -gt 0 ]; do done if [ -z "$mongodb_version" ]; then - export mongodb_version=6.0 + export mongodb_version=7.0 fi if [ -z "$mongdb_setup" ]; then @@ -40,7 +40,7 @@ psmdb_latest=$( wget -q --post-data "version=percona-server-mongodb-${mongodb_ve if [[ "$mongodb_version" == "4.4" ]]; then psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) else - psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep focal-minimal) + psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep jammy-minimal) fi echo "Downloading ${mongodb_version} ..." @@ -51,6 +51,19 @@ export extracted_folder_name=$(ls | grep percona-server-mongodb) echo "Extracted folder name ${extracted_folder_name}" mv ${extracted_folder_name} psmdb_${mongodb_version} +## Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +psmdb_tarball=$(curl -s "https://www.mongodb.com/try/download/community" | \ + grep -oP "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2004-${mongodb_version//./\\.}\.\d+\.tgz" | \ + sort -V | tail -n 1) + +echo "Downloading ${mongodb_version} from ${psmdb_tarball}..." +wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} +tar -xvf percona_server_mongodb.tar.gz + +export extracted_folder_name=$(ls | grep mongodb-linux) +echo "Extracted folder name ${extracted_folder_name}" +mv ${extracted_folder_name} psmdb_${mongodb_version} + # TODO: refactor if to match range of versions 6.0+ if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" || "$mongodb_version" == "8.0" ]]; then ### PSMDB 6+ requires "percona-mongodb-mongosh" additionally diff --git a/pmm_qa/mlaunch_setup.yml b/pmm_qa/mlaunch_psmdb_setup.yml similarity index 94% rename from pmm_qa/mlaunch_setup.yml rename to pmm_qa/mlaunch_psmdb_setup.yml index b42ff62..492aeb9 100644 --- a/pmm_qa/mlaunch_setup.yml +++ b/pmm_qa/mlaunch_psmdb_setup.yml @@ -34,7 +34,7 @@ - name: Copy all required Artifacts to the docker psmdb_container shell: "{{ item }}" with_items: - - docker cp ./mlaunch_setup.sh {{ psmdb_container }}:/ + - docker cp ./mlaunch_psmdb_setup.sh {{ psmdb_container }}:/ - docker cp ./pmm3-client-setup.sh {{ psmdb_container }}:/ - docker exec {{ psmdb_container }} apt-get update - docker exec {{ psmdb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq python3 pip @@ -54,7 +54,7 @@ - name: Setup psmdb for monitoring shell: "{{ item }}" with_items: - - docker exec {{ psmdb_container }} bash -x ./mlaunch_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + - docker exec {{ psmdb_container }} bash -x ./mlaunch_psmdb_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log - name: Setup Load Running Docker Container shell: "{{ item }}" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 4e11e40..01fd1ba 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -13,7 +13,11 @@ "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, - "MLAUNCH": { + "MLAUNCH_PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} + }, + "MLAUNCH_MODB": { "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} }, @@ -418,7 +422,7 @@ def setup_external(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) -def setup_mlaunch(db_type, db_version=None, db_config=None, args=None): +def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() if container_name is None and args.pmm_server_ip is None: @@ -441,7 +445,35 @@ def setup_mlaunch(db_type, db_version=None, db_config=None, args=None): } # Ansible playbook filename - playbook_filename = 'mlaunch_setup.yml' + playbook_filename = 'mlaunch_psmdb_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + +def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + modb_version = os.getenv('MODB_VERSION') or db_version or \ + database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'MODB_VERSION': modb_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MODB_CONTAINER': 'modb_pmm_' + str(modb_version), + 'MODB_SETUP': get_value('SETUP_TYPE', db_type, args, db_config), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'mlaunch_modb_setup.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) @@ -811,8 +843,10 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_ssl_pdpgsql(db_type, db_version, db_config, args) elif db_type == 'SSL_PSMDB': setup_ssl_psmdb(db_type, db_version, db_config, args) - elif db_type == 'MLAUNCH': - setup_mlaunch(db_type, db_version, db_config, args) + elif db_type == 'MLAUNCH_PSMDB': + setup_mlaunch_psmdb(db_type, db_version, db_config, args) + elif db_type == 'MLAUNCH_MODB': + setup_mlaunch_modb(db_type, db_version, db_config, args) elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args)