From ac950fe44857c82b6071253c11a4623991f28b94 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Tue, 19 Oct 2021 13:25:26 -0600 Subject: [PATCH 1/6] UDX-6042 Added shutdown and start * Added Shutdown script * Added Start script --- k8_cortx_cloud/destroy-cortx-cloud.sh | 3 -- k8_cortx_cloud/shutdown-cortx-cloud.sh | 55 +++++++++++++++++++ k8_cortx_cloud/start-cortx-cloud.sh | 75 ++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100755 k8_cortx_cloud/shutdown-cortx-cloud.sh create mode 100755 k8_cortx_cloud/start-cortx-cloud.sh diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index f73d880c..da54fa21 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -6,9 +6,6 @@ pvc_zookeeper_filter="zookeeper" pv_filter="pvc" openldap_pvc="openldap-data" -################################################################# -# Create files that contain disk partitions on the worker nodes -################################################################# function parseSolution() { echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" diff --git a/k8_cortx_cloud/shutdown-cortx-cloud.sh b/k8_cortx_cloud/shutdown-cortx-cloud.sh new file mode 100755 index 00000000..4817be0b --- /dev/null +++ b/k8_cortx_cloud/shutdown-cortx-cloud.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +function parseSolution() +{ + echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" +} + +namespace=$(parseSolution 'solution.namespace') +namespace=$(echo $namespace | cut -f2 -d'>') + +printf "########################################################\n" +printf "# Shutdown CORTX Data \n" +printf "########################################################\n" + +while IFS= read -r line; do + IFS=" " read -r -a deployments <<< "$line" + kubectl scale deploy "${deployments[0]}" --replicas 0 --namespace=$namespace +done <<< "$(kubectl get deployments --namespace=$namespace | grep 'cortx-control-pod')" + +printf "\nWait for CORTX Control to be shutdown" +while true; do + output=$(kubectl get pods --namespace=$namespace | grep 'cortx-control-pod-') + if [[ "$output" == "" ]]; then + break + else + printf "." + fi + sleep 1s +done +printf "\n\n" +printf "All CORTX Control pods have been shutdown" +printf "\n\n" + +printf "########################################################\n" +printf "# Shutdown CORTX Data \n" +printf "########################################################\n" + +while IFS= read -r line; do + IFS=" " read -r -a deployments <<< "$line" + kubectl scale deploy "${deployments[0]}" --replicas 0 --namespace=$namespace +done <<< "$(kubectl get deployments --namespace=$namespace | grep 'cortx-data-pod-')" + +printf "\nWait for CORTX Data to be shutdown" +while true; do + output=$(kubectl get pods --namespace=$namespace | grep 'cortx-data-pod-') + if [[ "$output" == "" ]]; then + break + else + printf "." + fi + sleep 1s +done +printf "\n\n" +printf "All CORTX Data pods have been shutdown" +printf "\n\n" \ No newline at end of file diff --git a/k8_cortx_cloud/start-cortx-cloud.sh b/k8_cortx_cloud/start-cortx-cloud.sh new file mode 100755 index 00000000..359702fb --- /dev/null +++ b/k8_cortx_cloud/start-cortx-cloud.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +function parseSolution() +{ + echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" +} + +namespace=$(parseSolution 'solution.namespace') +namespace=$(echo $namespace | cut -f2 -d'>') + +printf "########################################################\n" +printf "# Start CORTX Control \n" +printf "########################################################\n" +num_nodes=0 +while IFS= read -r line; do + IFS=" " read -r -a deployments <<< "$line" + kubectl scale deploy "${deployments[0]}" --replicas 1 --namespace=$namespace + num_nodes=$((num_nodes+1)) +done <<< "$(kubectl get deployments --namespace=$namespace | grep 'cortx-control-pod')" + +printf "\nWait for CORTX Control to be ready" +while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[1]}" + if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + break + fi + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-pod-')" + + if [[ $num_nodes -eq $count ]]; then + break + else + printf "." + fi + sleep 1s +done +printf "\n\n" +printf "All CORTX Control pods have been started" +printf "\n\n" + +printf "########################################################\n" +printf "# Start CORTX Data \n" +printf "########################################################\n" +num_nodes=0 +while IFS= read -r line; do + IFS=" " read -r -a deployments <<< "$line" + kubectl scale deploy "${deployments[0]}" --replicas 1 --namespace=$namespace + num_nodes=$((num_nodes+1)) +done <<< "$(kubectl get deployments --namespace=$namespace | grep 'cortx-data-pod-')" + +printf "\nWait for CORTX Data to be ready" +while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[1]}" + if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + break + fi + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-pod-')" + + if [[ $num_nodes -eq $count ]]; then + break + else + printf "." + fi + sleep 1s +done +printf "\n\n" +printf "All CORTX Data pods have been started" +printf "\n\n" \ No newline at end of file From d92d8f5e97c28974967c28da07ee1d723adaae4b Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Tue, 19 Oct 2021 14:12:07 -0600 Subject: [PATCH 2/6] UDX-6042 Added empty status script --- k8_cortx_cloud/status-cortx-cloud.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 k8_cortx_cloud/status-cortx-cloud.sh diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh new file mode 100755 index 00000000..b7e01557 --- /dev/null +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +function parseSolution() +{ + echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" +} + +namespace=$(parseSolution 'solution.namespace') +namespace=$(echo $namespace | cut -f2 -d'>') + From 32ddae32c8dc9b4b7bc7146d87068ce4b990b55d Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Tue, 19 Oct 2021 16:20:06 -0600 Subject: [PATCH 3/6] UDX-6042 status script control and data * Check deployments * Check pods * Check services --- k8_cortx_cloud/status-cortx-cloud.sh | 253 +++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh index b7e01557..bc816dd6 100755 --- a/k8_cortx_cloud/status-cortx-cloud.sh +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -1,5 +1,11 @@ #!/bin/bash +FAILED='\033[0;31m' #RED +PASSED='\033[0;32m' #GREEN +ALERT='\033[0;33m' #YELLOW +INFO='\033[0;36m' #CYAN +NC='\033[0m' #NO COLOUR + function parseSolution() { echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" @@ -8,3 +14,250 @@ function parseSolution() namespace=$(parseSolution 'solution.namespace') namespace=$(echo $namespace | cut -f2 -d'>') +######################################################################################### +# CORTX Control +######################################################################################### +num_nodes=1 +printf "${ALERT}######################################################${NC}\n" +printf "${ALERT}# CORTX Control ${NC}\n" +printf "${ALERT}######################################################${NC}\n" +# Check deployments +count=0 +printf "${INFO}| Checking Deployments |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a deployment_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${deployment_status[1]}" + if [[ "${deployment_status[0]}" != "" ]]; then + printf "${deployment_status[0]}..." + if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get deployments --namespace=$namespace | grep 'cortx-control-pod')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check pods +count=0 +printf "${INFO}| Checking Pods |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[1]}" + if [[ "${pod_status[0]}" != "" ]]; then + printf "${pod_status[0]}..." + if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi + +done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-pod-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services headless +count=0 +printf "${INFO}| Checking Services: Headless |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[0]}" != "" ]]; then + printf "${service_status[0]}..." + if [[ "${service_status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi + +done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-control-headless-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services cluster IP +count=0 +printf "${INFO}| Checking Services: Cluster IP |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[0]}" != "" ]]; then + printf "${service_status[0]}..." + if [[ "${service_status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + fi + count=$((count+1)) + fi + +done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-control-clusterip-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services node port +count=0 +printf "${INFO}| Checking Services: Node Port |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[0]}" != "" ]]; then + printf "${service_status[0]}..." + if [[ "${service_status[1]}" != "NodePort" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi + +done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-control-nodeport-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +######################################################################################### +# CORTX Data +######################################################################################### +nodes_names=$(parseSolution 'solution.nodes.node*.name') +num_nodes=$(echo $nodes_names | grep -o '>' | wc -l) + +printf "${ALERT}######################################################${NC}\n" +printf "${ALERT}# CORTX Data ${NC}\n" +printf "${ALERT}######################################################${NC}\n" +# Check deployments +count=0 +printf "${INFO}| Checking Deployments |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a deployment_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${deployment_status[1]}" + if [[ "${deployment_status[0]}" != "" ]]; then + printf "${deployment_status[0]}..." + if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get deployments --namespace=$namespace | grep 'cortx-data-pod-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check pods +count=0 +printf "${INFO}| Checking Pods |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[1]}" + if [[ "${pod_status[0]}" != "" ]]; then + printf "${pod_status[0]}..." + if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi + +done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-pod-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services headless +count=0 +printf "${INFO}| Checking Services: Headless |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[0]}" != "" ]]; then + printf "${service_status[0]}..." + if [[ "${service_status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi + +done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-data-headless-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services cluster IP +count=0 +printf "${INFO}| Checking Services: Cluster IP |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[0]}" != "" ]]; then + printf "${service_status[0]}..." + if [[ "${service_status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + fi + count=$((count+1)) + fi + +done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-data-clusterip-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services load balance +count=0 +printf "${INFO}| Checking Services: Load Balancer |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[0]}" != "" ]]; then + printf "${service_status[0]}..." + if [[ "${service_status[1]}" != "LoadBalancer" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi + +done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-data-loadbal-')" + +if [[ $num_nodes -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi From 750d7172bbd3780d3a005172a2ab976604cd4e94 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Wed, 20 Oct 2021 12:08:39 -0600 Subject: [PATCH 4/6] Integration updates UDX-6412, UDX-6410, UDX-6411 --- k8_cortx_cloud/README.txt | 7 +- .../templates/openldap-statefulset.yaml | 2 +- .../openldap/values.yaml | 3 +- .../templates/cortx-gluster.yaml | 2 +- .../cortx-gluster/values.yaml | 1 + k8_cortx_cloud/deploy-cortx-cloud.sh | 32 +- k8_cortx_cloud/destroy-cortx-cloud.sh | 13 +- k8_cortx_cloud/prereq-deploy-cortx-cloud.sh | 93 +++- k8_cortx_cloud/shutdown-cortx-cloud.sh | 11 +- k8_cortx_cloud/solution.yaml | 7 +- k8_cortx_cloud/start-cortx-cloud.sh | 11 +- k8_cortx_cloud/status-cortx-cloud.sh | 477 ++++++++++++++++-- 12 files changed, 581 insertions(+), 78 deletions(-) diff --git a/k8_cortx_cloud/README.txt b/k8_cortx_cloud/README.txt index 7e468b7c..37f99992 100644 --- a/k8_cortx_cloud/README.txt +++ b/k8_cortx_cloud/README.txt @@ -99,4 +99,9 @@ solution: cortxcontrol: centos:7 cortxdataprov: centos:7 cortxdata: centos:7 - cortxsupport: centos:7 \ No newline at end of file + openldap: ghcr.io/seagate/symas-openldap:standalone + consul: hashicorp/consul:1.10.2 + kafka: bitnami/kafka + zookeeper: bitnami/zookeeper + gluster: docker.io/gluster/gluster-centos + rancher: rancher/local-path-provisioner:v0.0.20 \ No newline at end of file diff --git a/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/templates/openldap-statefulset.yaml b/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/templates/openldap-statefulset.yaml index 4bbab4f9..f7c30c6f 100644 --- a/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/templates/openldap-statefulset.yaml +++ b/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/templates/openldap-statefulset.yaml @@ -24,7 +24,7 @@ spec: name: openldap-connect containers: - name: openldap - image: ghcr.io/seagate/symas-openldap:standalone + image: {{ .Values.openldap.image }} imagePullPolicy: IfNotPresent volumeMounts: - name: etc-3rd-party diff --git a/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/values.yaml b/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/values.yaml index a438a951..368ef4f5 100644 --- a/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/values.yaml +++ b/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap/values.yaml @@ -5,4 +5,5 @@ openldap: storagesize: 5Gi nodelistinfo: node-list-info.txt numreplicas: 3 - password: seagate1 \ No newline at end of file + password: seagate1 + image: ghcr.io/seagate/symas-openldap:standalone \ No newline at end of file diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/templates/cortx-gluster.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/templates/cortx-gluster.yaml index dd656fd3..ac362b93 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/templates/cortx-gluster.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/templates/cortx-gluster.yaml @@ -18,7 +18,7 @@ spec: app: {{ .Values.cortxgluster.name }} spec: containers: - - image: docker.io/gluster/gluster-centos + - image: {{ .Values.cortxgluster.image }} imagePullPolicy: IfNotPresent name: {{ .Values.cortxgluster.name }} livenessProbe: diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/values.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/values.yaml index d0e37b8e..ea38b140 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/values.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-gluster/values.yaml @@ -4,6 +4,7 @@ cortxgluster: nodename: node-1 storagesize: 1Gi storageclass: cortx-gluster-storage + image: docker.io/gluster/gluster-centos service: name: cortx-gluster-svc pv: diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index f97d2a62..fe10e619 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -1,7 +1,16 @@ #!/bin/bash +solution_yaml=${1:-'solution.yaml'} storage_class='local-path' +# Check if the file exists +if [ ! -f $solution_yaml ] +then + echo "ERROR: $solution_yaml does not exist" + exit 1 +fi + + # Delete old "node-list-info.txt" file find $(pwd)/cortx-cloud-3rd-party-pkg/openldap -name "node-list-info*" -delete @@ -33,12 +42,12 @@ printf "Number of worker nodes detected: $num_worker_nodes\n" function parseSolution() { - echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" + echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" } function extractBlock() { - echo "$(./parse_scripts/yaml_extract_block.sh solution.yaml $1)" + echo "$(./parse_scripts/yaml_extract_block.sh $solution_yaml $1)" } namespace=$(parseSolution 'solution.namespace') @@ -120,7 +129,7 @@ do if [[ "${#parsed_dev_array[@]}" != "${#parsed_size_array[@]}" ]] then printf "\nStorage sizes are not defined for all of the storage devices\n" - printf "in the 'solution.yaml' file\n" + printf "in the $solution_yaml file\n" exit 1 fi @@ -181,8 +190,12 @@ then kubectl create -f cortx-cloud-3rd-party-pkg/local-path-storage.yaml fi +image=$(parseSolution 'solution.images.consul') +image=$(echo $image | cut -f2 -d'>') + helm install "consul" hashicorp/consul \ --set global.name="consul" \ + --set global.image=$image \ --set server.storageClass=$storage_class \ --set server.replicas=$num_consul_replicas @@ -192,6 +205,8 @@ printf "######################################################\n" openldap_password=$(parseSolution 'solution.3rdparty.openldap.password') openldap_password=$(echo $openldap_password | cut -f2 -d'>') +image=$(parseSolution 'solution.images.openldap') +image=$(echo $image | cut -f2 -d'>') helm install "openldap" cortx-cloud-3rd-party-pkg/openldap \ --set openldap.servicename="openldap-svc" \ @@ -199,7 +214,8 @@ helm install "openldap" cortx-cloud-3rd-party-pkg/openldap \ --set openldap.storagesize="5Gi" \ --set openldap.nodelistinfo="node-list-info.txt" \ --set openldap.numreplicas=$num_openldap_replicas \ - --set openldap.password=$openldap_password + --set openldap.password=$openldap_password \ + --set openldap.image=$image # Wait for all openLDAP pods to be ready printf "\nWait for openLDAP PODs to be ready" @@ -349,10 +365,14 @@ printf "########################################################\n" first_node_name=${node_name_list[0]} first_node_selector=${node_selector_list[0]} +image=$(parseSolution 'solution.images.gluster') +image=$(echo $image | cut -f2 -d'>') + helm install "cortx-gluster-$first_node_name" cortx-cloud-helm-pkg/cortx-gluster \ --set cortxgluster.name="gluster-$first_node_name" \ --set cortxgluster.nodename=$first_node_selector \ --set cortxgluster.service.name="cortx-gluster-svc-$first_node_name" \ + --set cortxgluster.image=$image \ --set cortxgluster.storagesize="1Gi" \ --set cortxgluster.storageclass="cortx-gluster-storage" \ --set cortxgluster.pv.path=$gluster_vol \ @@ -663,14 +683,14 @@ printf "########################################################\n" # in the "auto-gen-secret" folder secret_auto_gen_path="$cfgmap_path/auto-gen-secret" mkdir -p $secret_auto_gen_path -output=$(./parse_scripts/parse_yaml.sh solution.yaml "solution.secrets.name") +output=$(./parse_scripts/parse_yaml.sh $solution_yaml "solution.secrets.name") IFS=';' read -r -a parsed_secret_name_array <<< "$output" for secret_name in "${parsed_secret_name_array[@]}" do secret_fname=$(echo $secret_name | cut -f2 -d'>') yaml_content_path=$(echo $secret_name | cut -f1 -d'>') yaml_content_path=${yaml_content_path/.name/".content"} - secrets="$(./parse_scripts/yaml_extract_block.sh solution.yaml $yaml_content_path 2)" + secrets="$(./parse_scripts/yaml_extract_block.sh $solution_yaml $yaml_content_path 2)" new_secret_gen_file="$secret_auto_gen_path/$secret_fname.yaml" cp "$cfgmap_path/templates/secret-template.yaml" $new_secret_gen_file diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index da54fa21..44dc31a3 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -1,5 +1,14 @@ #!/bin/bash +solution_yaml=${1:-'solution.yaml'} + +# Check if the file exists +if [ ! -f $solution_yaml ] +then + echo "ERROR: $solution_yaml does not exist" + exit 1 +fi + pvc_consul_filter="data-default-consul" pvc_kafka_filter="kafka" pvc_zookeeper_filter="zookeeper" @@ -8,7 +17,7 @@ openldap_pvc="openldap-data" function parseSolution() { - echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" + echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" } namespace=$(parseSolution 'solution.namespace') @@ -235,7 +244,7 @@ helm uninstall "openldap" printf "########################################################\n" printf "# Delete Secrets #\n" printf "########################################################\n" -output=$(./parse_scripts/parse_yaml.sh solution.yaml "solution.secrets*.name") +output=$(./parse_scripts/parse_yaml.sh $solution_yaml "solution.secrets*.name") IFS=';' read -r -a parsed_secret_name_array <<< "$output" for secret_name in "${parsed_secret_name_array[@]}" do diff --git a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh index b01c84e0..fdf17a7b 100755 --- a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh @@ -1,6 +1,20 @@ #!/bin/bash disk_partition=$1 +solution_yaml=${2:-'solution.yaml'} + +# Check if the file exists +if [ ! -f $solution_yaml ] +then + echo "ERROR: $solution_yaml does not exist" + exit 1 +fi + +function parseSolution() +{ + echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" +} + fs_mount_path="/mnt/fs-local-volume" if [[ "$disk_partition" == "" ]] @@ -21,15 +35,69 @@ printf "####################################################\n" printf "# Pull required docker images \n" printf "####################################################\n" # pull docker 3rd party images -docker pull bitnami/kafka -docker pull docker.io/gluster/gluster-centos -docker pull rancher/local-path-provisioner:v0.0.20 -docker pull bitnami/zookeeper -docker pull hashicorp/consul:1.10.2 -docker pull busybox -#Pull cortx-all docker image -docker pull ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci +image=$(parseSolution 'solution.images.cortxcontrolprov') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.cortxcontrol') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.cortxdataprov') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.cortxdata') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.openldap') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.consul') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.kafka') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.zookeeper') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.gluster') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +image=$(parseSolution 'solution.images.rancher') +image=$(echo $image | cut -f2 -d'>') +if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image +fi + +# Pull the latest busybox image +docker pull busybox printf "####################################################\n" printf "# Clean up \n" @@ -47,8 +115,13 @@ printf "# Prep for CORTX deployment \n" printf "####################################################\n" # Prep for Rancher Local Path Provisioner deployment mkdir -p $fs_mount_path/local-path-provisioner -echo y | mkfs.ext4 $disk_partition -mount -t ext4 $disk_partition $fs_mount_path + +if [[ $(findmnt -m $fs_mount_path) ]];then + echo "$fs_mount_path already mounted..." +else + echo y | mkfs.ext4 $disk_partition + mount -t ext4 $disk_partition $fs_mount_path +fi # Prep for GlusterFS deployment yum install glusterfs-fuse -y diff --git a/k8_cortx_cloud/shutdown-cortx-cloud.sh b/k8_cortx_cloud/shutdown-cortx-cloud.sh index 4817be0b..5272924e 100755 --- a/k8_cortx_cloud/shutdown-cortx-cloud.sh +++ b/k8_cortx_cloud/shutdown-cortx-cloud.sh @@ -1,8 +1,17 @@ #!/bin/bash +solution_yaml=${1:-'solution.yaml'} + +# Check if the file exists +if [ ! -f $solution_yaml ] +then + echo "ERROR: $solution_yaml does not exist" + exit 1 +fi + function parseSolution() { - echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" + echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" } namespace=$(parseSolution 'solution.namespace') diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index 7aa0ac2b..41a32f65 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -15,7 +15,12 @@ solution: cortxcontrol: centos:7 cortxdataprov: centos:7 cortxdata: centos:7 - cortxsupport: centos:7 + openldap: ghcr.io/seagate/symas-openldap:standalone + consul: hashicorp/consul:1.10.0 + kafka: bitnami/kafka + zookeeper: bitnami/zookeeper + gluster: docker.io/gluster/gluster-centos + rancher: rancher/local-path-provisioner:v0.0.20 3rdparty: openldap: password: seagate1 diff --git a/k8_cortx_cloud/start-cortx-cloud.sh b/k8_cortx_cloud/start-cortx-cloud.sh index 359702fb..67e2271c 100755 --- a/k8_cortx_cloud/start-cortx-cloud.sh +++ b/k8_cortx_cloud/start-cortx-cloud.sh @@ -1,8 +1,17 @@ #!/bin/bash +solution_yaml=${1:-'solution.yaml'} + +# Check if the file exists +if [ ! -f $solution_yaml ] +then + echo "ERROR: $solution_yaml does not exist" + exit 1 +fi + function parseSolution() { - echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" + echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" } namespace=$(parseSolution 'solution.namespace') diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh index bc816dd6..d5d42a67 100755 --- a/k8_cortx_cloud/status-cortx-cloud.sh +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -1,5 +1,14 @@ #!/bin/bash +solution_yaml=${1:-'solution.yaml'} + +# Check if the file exists +if [ ! -f $solution_yaml ] +then + echo "ERROR: $solution_yaml does not exist" + exit 1 +fi + FAILED='\033[0;31m' #RED PASSED='\033[0;32m' #GREEN ALERT='\033[0;33m' #YELLOW @@ -8,7 +17,7 @@ NC='\033[0m' #NO COLOUR function parseSolution() { - echo "$(./parse_scripts/parse_yaml.sh solution.yaml $1)" + echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" } namespace=$(parseSolution 'solution.namespace') @@ -25,10 +34,10 @@ printf "${ALERT}######################################################${NC}\n" count=0 printf "${INFO}| Checking Deployments |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a deployment_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${deployment_status[1]}" - if [[ "${deployment_status[0]}" != "" ]]; then - printf "${deployment_status[0]}..." + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then printf "${FAILED}FAILED${NC}\n" else @@ -48,18 +57,17 @@ fi count=0 printf "${INFO}| Checking Pods |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[1]}" - if [[ "${pod_status[0]}" != "" ]]; then - printf "${pod_status[0]}..." - if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" count=$((count+1)) fi fi - done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-pod-')" if [[ $num_nodes -eq $count ]]; then @@ -72,17 +80,16 @@ fi count=0 printf "${INFO}| Checking Services: Headless |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[0]}" != "" ]]; then - printf "${service_status[0]}..." - if [[ "${service_status[1]}" != "ClusterIP" ]]; then + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" count=$((count+1)) fi fi - done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-control-headless-')" if [[ $num_nodes -eq $count ]]; then @@ -95,17 +102,16 @@ fi count=0 printf "${INFO}| Checking Services: Cluster IP |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[0]}" != "" ]]; then - printf "${service_status[0]}..." - if [[ "${service_status[1]}" != "ClusterIP" ]]; then + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) fi - count=$((count+1)) fi - done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-control-clusterip-')" if [[ $num_nodes -eq $count ]]; then @@ -118,17 +124,16 @@ fi count=0 printf "${INFO}| Checking Services: Node Port |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[0]}" != "" ]]; then - printf "${service_status[0]}..." - if [[ "${service_status[1]}" != "NodePort" ]]; then + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "NodePort" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" count=$((count+1)) fi fi - done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-control-nodeport-')" if [[ $num_nodes -eq $count ]]; then @@ -150,10 +155,10 @@ printf "${ALERT}######################################################${NC}\n" count=0 printf "${INFO}| Checking Deployments |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a deployment_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${deployment_status[1]}" - if [[ "${deployment_status[0]}" != "" ]]; then - printf "${deployment_status[0]}..." + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then printf "${FAILED}FAILED${NC}\n" else @@ -173,18 +178,17 @@ fi count=0 printf "${INFO}| Checking Pods |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[1]}" - if [[ "${pod_status[0]}" != "" ]]; then - printf "${pod_status[0]}..." - if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" count=$((count+1)) fi fi - done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-pod-')" if [[ $num_nodes -eq $count ]]; then @@ -197,17 +201,16 @@ fi count=0 printf "${INFO}| Checking Services: Headless |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[0]}" != "" ]]; then - printf "${service_status[0]}..." - if [[ "${service_status[1]}" != "ClusterIP" ]]; then + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" count=$((count+1)) fi fi - done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-data-headless-')" if [[ $num_nodes -eq $count ]]; then @@ -220,17 +223,16 @@ fi count=0 printf "${INFO}| Checking Services: Cluster IP |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[0]}" != "" ]]; then - printf "${service_status[0]}..." - if [[ "${service_status[1]}" != "ClusterIP" ]]; then + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) fi - count=$((count+1)) fi - done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-data-clusterip-')" if [[ $num_nodes -eq $count ]]; then @@ -243,17 +245,16 @@ fi count=0 printf "${INFO}| Checking Services: Load Balancer |${NC}\n" while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[0]}" != "" ]]; then - printf "${service_status[0]}..." - if [[ "${service_status[1]}" != "LoadBalancer" ]]; then + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "LoadBalancer" ]]; then printf "${FAILED}FAILED${NC}\n" else printf "${PASSED}PASSED${NC}\n" count=$((count+1)) fi fi - done <<< "$(kubectl get services --namespace=$namespace | grep 'cortx-data-loadbal-')" if [[ $num_nodes -eq $count ]]; then @@ -261,3 +262,373 @@ if [[ $num_nodes -eq $count ]]; then else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" fi + +######################################################################################### +# 3rd Party +######################################################################################### +nodes_names=$(parseSolution 'solution.nodes.node*.name') +num_nodes=$(echo $nodes_names | grep -o '>' | wc -l) +max_replicas=3 +num_replicas=$num_nodes +if [[ "$num_nodes" -gt "$max_replicas" ]]; then + num_replicas=$max_replicas +fi +printf "${ALERT}######################################################${NC}\n" +printf "${ALERT}# 3rd Party ${NC}\n" +printf "${ALERT}######################################################${NC}\n" +printf "${ALERT}### OpenLDAP${NC}\n" +# Check StatefulSet +num_items=1 +count=0 +printf "${INFO}| Checking StatefulSet |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get statefulsets | grep 'openldap')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check Pods +num_items=$num_replicas +count=0 +printf "${INFO}| Checking Pods |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pods | grep 'openldap-')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services cluster IP +num_items=1 +count=0 +printf "${INFO}| Checking Services: Cluster IP |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'openldap-svc')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +printf "${ALERT}### Kafka${NC}\n" +# Check StatefulSet +num_items=1 +count=0 +printf "${INFO}| Checking StatefulSet |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get statefulsets | grep 'kafka')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check Pods +num_items=$num_replicas +count=0 +printf "${INFO}| Checking Pods |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pods | grep 'kafka-')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services cluster IP +num_items=1 +count=0 +printf "${INFO}| Checking Services: Cluster IP |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'kafka' | grep -v 'kafka-headless')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services headless +num_items=1 +count=0 +printf "${INFO}| Checking Services: Headless |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'kafka-headless')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +printf "${ALERT}### Zookeeper${NC}\n" +# Check StatefulSet +num_items=1 +count=0 +printf "${INFO}| Checking StatefulSet |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get statefulsets | grep 'zookeeper')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check Pods +num_items=$num_replicas +count=0 +printf "${INFO}| Checking Pods |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pods | grep 'zookeeper-')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services cluster IP +num_items=1 +count=0 +printf "${INFO}| Checking Services: Cluster IP |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'zookeeper' | grep -v 'zookeeper-headless')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services headless +num_items=1 +count=0 +printf "${INFO}| Checking Services: Headless |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'zookeeper-headless')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +printf "${ALERT}### Consul${NC}\n" +# Check StatefulSet +num_items=1 +count=0 +printf "${INFO}| Checking StatefulSet |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get statefulsets | grep 'consul')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check Pods +num_items=$(($num_replicas*2)) +count=0 +printf "${INFO}| Checking Pods |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + IFS="/" read -r -a ready_status <<< "${status[1]}" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pods | grep 'consul-')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services cluster IP +num_items=2 +count=0 +printf "${INFO}| Checking Services: Cluster IP |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'consul' | grep -v 'consul-server')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check services headless +num_items=1 +count=0 +printf "${INFO}| Checking Services: Headless |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "${status[0]}..." + if [[ "${status[1]}" != "ClusterIP" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get services | grep 'consul-server')" + +if [[ $num_items -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi \ No newline at end of file From 311f8669e5d142d6dde0564a17f607778aae08a4 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Wed, 20 Oct 2021 13:45:43 -0600 Subject: [PATCH 5/6] UDX-6042 Updated status script --- k8_cortx_cloud/status-cortx-cloud.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh index d5d42a67..2e1dce82 100755 --- a/k8_cortx_cloud/status-cortx-cloud.sh +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -564,7 +564,7 @@ else fi # Check Pods -num_items=$(($num_replicas*2)) +num_items=$(($num_replicas)) count=0 printf "${INFO}| Checking Pods |${NC}\n" while IFS= read -r line; do @@ -579,7 +579,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pods | grep 'consul-')" +done <<< "$(kubectl get pods | grep 'consul-server-')" if [[ $num_items -eq $count ]]; then printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" From d8f8439c8ebe698c1fa1d9a26b7dff61ce070973 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Wed, 20 Oct 2021 15:20:16 -0600 Subject: [PATCH 6/6] UDX-6435 Glusterfs shared volume is getting full and not getting cleaned up --- k8_cortx_cloud/destroy-cortx-cloud.sh | 3 ++- k8_cortx_cloud/prereq-deploy-cortx-cloud.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index 44dc31a3..5871f4f4 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -139,7 +139,8 @@ do printf "=================================================================================\n" printf "Stop and delete GlusterFS volume: $gluster_node_name \n" printf "=================================================================================\n" - + kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ + 'rm -rf /etc/gluster/* /etc/gluster/.glusterfs/' if [[ "$count" == 0 ]]; then first_gluster_node_name=$gluster_node_name echo y | kubectl exec --namespace=$namespace -i $gluster_node_name -- gluster volume stop $gluster_vol diff --git a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh index fdf17a7b..292acc86 100755 --- a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh @@ -126,6 +126,7 @@ fi # Prep for GlusterFS deployment yum install glusterfs-fuse -y mkdir -p $fs_mount_path/etc/gluster +mkdir -p $fs_mount_path/etc/gluster/var/log/cortx mkdir -p $fs_mount_path/var/log/glusterfs mkdir -p $fs_mount_path/var/lib/glusterd