From e0590056d470eb1731fceef273b2bb90b5feefa6 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Wed, 27 Oct 2021 10:48:55 -0600 Subject: [PATCH 01/16] Fixed issue in error reporting in parse_yaml.sh --- k8_cortx_cloud/parse_scripts/parse_yaml.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8_cortx_cloud/parse_scripts/parse_yaml.sh b/k8_cortx_cloud/parse_scripts/parse_yaml.sh index 8163580b..3654c21c 100755 --- a/k8_cortx_cloud/parse_scripts/parse_yaml.sh +++ b/k8_cortx_cloud/parse_scripts/parse_yaml.sh @@ -36,7 +36,7 @@ fi # Check if the file exists if [ ! -f $INPUT_YAML_FILE ] then - echo "ERROR: $YAML_FILE_TO_MOD does not exist" + echo "ERROR: $INPUT_YAML_FILE does not exist" exit 1 fi From d3b5f3fa1c9c077aa6cfb19ee3e0370eb465c1f5 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 27 Oct 2021 11:20:04 -0600 Subject: [PATCH 02/16] UDX-6040_helm_and_script_updates - create functions for all deployments in the deploy script --- k8_cortx_cloud/deploy-cortx-cloud.sh | 1733 ++++++++++++++------------ 1 file changed, 934 insertions(+), 799 deletions(-) diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 5aa01498..0e7dd621 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -19,6 +19,9 @@ max_consul_inst=3 max_kafka_inst=3 num_openldap_replicas=0 # Default the number of actual openldap instances num_worker_nodes=0 +# Create a file consist of a list of node info and up to 'max_openldap_inst' +# number of nodes. This file is used by OpenLDAP helm chart and will be deleted +# at the end of this script. while IFS= read -r line; do IFS=" " read -r -a node_name <<< "$line" if [[ "$node_name" != "NAME" ]]; then @@ -61,7 +64,8 @@ tainted_worker_node_list=[] num_tainted_worker_nodes=0 not_found_node_list=[] num_not_found_nodes=0 -# Validate solution yaml file doesn't have nodes that are tainted with "NoSchedule" +# Validate the solution file. Check that nodes listed in the solution file +# aren't tainted and allow scheduling. for parsed_var_val_element in "${parsed_var_val_array[@]}"; do node_name=$(echo $parsed_var_val_element | cut -f2 -d'>') @@ -75,7 +79,7 @@ do num_not_found_nodes=$((num_not_found_nodes+1)) fi done - +# Print a list of tainted nodes and nodes that don't exist in the cluster if [[ $num_tainted_worker_nodes -gt 0 || $num_not_found_nodes -gt 0 ]]; then echo "Can't deploy CORTX cloud." if [[ $num_tainted_worker_nodes -gt 0 ]]; then @@ -95,6 +99,9 @@ fi find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "mnt-blk-*" -delete find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "mnt-blk-*" -delete +# Create files consist of drives per node and files consist of drive sizes. +# These files are used by the helm charts to deploy cortx provisioners and +# cortx data. These file will be deleted at the end of this script. node_name_list=[] # short version. Ex: ssc-vm-g3-rhev4-1490 node_selector_list=[] # long version. Ex: ssc-vm-g3-rhev4-1490.colo.seagate.com count=0 @@ -160,6 +167,7 @@ do done done +# Create CORTX namespace if [[ "$namespace" != "default" ]]; then kubectl create namespace $namespace fi @@ -167,879 +175,1006 @@ fi ########################################################## # Deploy CORTX 3rd party ########################################################## +function deployRancherProvisioner() +{ + storage_class=$1 + storage_prov_path=$2 + # Add the HashiCorp Helm Repository: + helm repo add hashicorp https://helm.releases.hashicorp.com + if [[ $storage_class == "local-path" ]] + then + printf "Install Rancher Local Path Provisioner" + rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" + # Clean up auto gen Rancher Provisioner folder in case it still exists and was not + # clearned up previously by the destroy-cortx-cloud script. + rm -rf $rancher_prov_path + mkdir -p $rancher_prov_path + rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" + cp $(pwd)/cortx-cloud-3rd-party-pkg/templates/local-path-storage-template.yaml $rancher_prov_file + ./parse_scripts/subst.sh $rancher_prov_file "rancher.host_path" "$storage_prov_path/local-path-provisioner" + + kubectl create -f $rancher_prov_file + fi +} -printf "######################################################\n" -printf "# Deploy Consul \n" -printf "######################################################\n" -num_consul_replicas=$num_worker_nodes -if [[ "$num_worker_nodes" -gt "$max_consul_inst" ]]; then - num_consul_replicas=$max_consul_inst -fi - -# Extract storage provisioner path from the "solution.yaml" file -filter='solution.common.storage_provisioner_path' -parse_storage_prov_output=$(parseSolution $filter) -# Get the storage provisioner var from the tuple -storage_prov_path=$(echo $parse_storage_prov_output | cut -f2 -d'>') +function deployConsul() +{ + storage_class=$1 + num_consul_replicas=$2 + printf "######################################################\n" + printf "# Deploy Consul \n" + printf "######################################################\n" + image=$(parseSolution 'solution.images.consul') + image=$(echo $image | cut -f2 -d'>') + + helm install "consul" hashicorp/consul \ + --set global.name="consul" \ + --set global.image=$image \ + --set server.storageClass=$storage_class \ + --set server.replicas=$num_consul_replicas +} -# Add the HashiCorp Helm Repository: -helm repo add hashicorp https://helm.releases.hashicorp.com -if [[ $storage_class == "local-path" ]] -then - printf "Install Rancher Local Path Provisioner" - rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" - # Clean up auto gen Rancher Provisioner folder in case it still exists and was not - # clearned up previously by the destroy-cortx-cloud script. - rm -rf $rancher_prov_path - mkdir -p $rancher_prov_path - rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" - cp $(pwd)/cortx-cloud-3rd-party-pkg/templates/local-path-storage-template.yaml $rancher_prov_file - ./parse_scripts/subst.sh $rancher_prov_file "rancher.host_path" "$storage_prov_path/local-path-provisioner" - - kubectl create -f $rancher_prov_file -fi +function deployOpenLDAP() +{ + num_openldap_replicas=$1 + printf "######################################################\n" + printf "# Deploy openLDAP \n" + printf "######################################################\n" + openldap_password=$(parseSolution 'solution.secrets.content.openldap_admin_secret') + openldap_password=$(echo $openldap_password | cut -f2 -d'>') + image=$(parseSolution 'solution.images.openldap') + image=$(echo $image | cut -f2 -d'>') + + helm install "openldap" cortx-cloud-3rd-party-pkg/openldap \ + --set openldap.servicename="openldap-svc" \ + --set openldap.storageclass="openldap-local-storage" \ + --set openldap.storagesize="5Gi" \ + --set openldap.nodelistinfo="node-list-info.txt" \ + --set openldap.numreplicas=$num_openldap_replicas \ + --set openldap.password=$openldap_password \ + --set openldap.image=$image + + # Wait for all openLDAP pods to be ready + printf "\nWait for openLDAP PODs to be ready" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[2]}" + if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + break + fi + count=$((count+1)) + done <<< "$(kubectl get pods -A | grep 'openldap')" -image=$(parseSolution 'solution.images.consul') -image=$(echo $image | cut -f2 -d'>') - -helm install "consul" hashicorp/consul \ - --set global.name="consul" \ - --set global.image=$image \ - --set server.storageClass=$storage_class \ - --set server.replicas=$num_consul_replicas - -printf "######################################################\n" -printf "# Deploy openLDAP \n" -printf "######################################################\n" - -openldap_password=$(parseSolution 'solution.secrets.content.openldap_admin_secret') -openldap_password=$(echo $openldap_password | cut -f2 -d'>') -image=$(parseSolution 'solution.images.openldap') -image=$(echo $image | cut -f2 -d'>') - -helm install "openldap" cortx-cloud-3rd-party-pkg/openldap \ - --set openldap.servicename="openldap-svc" \ - --set openldap.storageclass="openldap-local-storage" \ - --set openldap.storagesize="5Gi" \ - --set openldap.nodelistinfo="node-list-info.txt" \ - --set openldap.numreplicas=$num_openldap_replicas \ - --set openldap.password=$openldap_password \ - --set openldap.image=$image - -# Wait for all openLDAP pods to be ready -printf "\nWait for openLDAP PODs to be ready" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[2]}" - if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + if [[ $count -eq $num_openldap_replicas ]]; then break + else + printf "." fi - count=$((count+1)) - done <<< "$(kubectl get pods -A | grep 'openldap')" + sleep 1s + done + printf "\n\n" - if [[ $count -eq $num_openldap_replicas ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" + printf "===========================================================\n" + printf "Setup OpenLDAP replication \n" + printf "===========================================================\n" + # Run replication script + ./cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh --rootdnpassword $openldap_password +} -printf "===========================================================\n" -printf "Setup OpenLDAP replication \n" -printf "===========================================================\n" -# Run replication script -./cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh --rootdnpassword $openldap_password +function deployZookeeper() +{ + storage_class=$1 + num_kafka_replicas=$2 + printf "######################################################\n" + printf "# Deploy Zookeeper \n" + printf "######################################################\n" + # Add Zookeeper and Kafka Repository + helm repo add bitnami https://charts.bitnami.com/bitnami + + helm install zookeeper bitnami/zookeeper \ + --set replicaCount=$num_kafka_replicas \ + --set auth.enabled=false \ + --set allowAnonymousLogin=true \ + --set global.storageClass=$storage_class +} -printf "######################################################\n" -printf "# Deploy Zookeeper \n" -printf "######################################################\n" -num_kafka_replicas=$num_worker_nodes -if [[ "$num_worker_nodes" -gt "$max_kafka_inst" ]]; then - num_kafka_replicas=$max_kafka_inst -fi +function deployKafka() +{ + num_worker_nodes=$1 + storage_class=$2 + num_kafka_replicas=$3 + printf "######################################################\n" + printf "# Deploy Kafka \n" + printf "######################################################\n" + helm install kafka bitnami/kafka \ + --set zookeeper.enabled=false \ + --set replicaCount=$num_kafka_replicas \ + --set externalZookeeper.servers=zookeeper.default.svc.cluster.local \ + --set global.storageClass=$storage_class \ + --set defaultReplicationFactor=$num_kafka_replicas \ + --set offsetTopicReplicationFactor=$num_kafka_replicas \ + --set transactionStateLogReplicationFactor=$num_kafka_replicas \ + --set auth.enabled=false \ + --set allowAnonymousLogin=true \ + --set deleteTopicEnable=true \ + --set transactionStateLogMinIsr=2 + + printf "\nWait for CORTX 3rd party to be ready" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[2]}" + if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + count=$((count+1)) + break + fi + done <<< "$(kubectl get pods -A | grep 'consul\|kafka\|openldap\|zookeeper')" -# Add Zookeeper and Kafka Repository -helm repo add bitnami https://charts.bitnami.com/bitnami - -helm install zookeeper bitnami/zookeeper \ - --set replicaCount=$num_kafka_replicas \ - --set auth.enabled=false \ - --set allowAnonymousLogin=true \ - --set global.storageClass=$storage_class - -printf "######################################################\n" -printf "# Deploy Kafka \n" -printf "######################################################\n" -helm install kafka bitnami/kafka \ - --set zookeeper.enabled=false \ - --set replicaCount=$num_kafka_replicas \ - --set externalZookeeper.servers=zookeeper.default.svc.cluster.local \ - --set global.storageClass=$storage_class \ - --set defaultReplicationFactor=$num_kafka_replicas \ - --set offsetTopicReplicationFactor=$num_kafka_replicas \ - --set transactionStateLogReplicationFactor=$num_kafka_replicas \ - --set auth.enabled=false \ - --set allowAnonymousLogin=true \ - --set deleteTopicEnable=true \ - --set transactionStateLogMinIsr=2 - -printf "\nWait for CORTX 3rd party to be ready" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[2]}" - if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then - count=$((count+1)) + if [[ $count -eq 0 ]]; then break + else + printf "." fi - done <<< "$(kubectl get pods -A | grep 'consul\|kafka\|openldap\|zookeeper')" - - if [[ $count -eq 0 ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" + sleep 1s + done + printf "\n\n" +} ########################################################## -# Deploy CORTX cloud +# CORTX cloud deploy functions ########################################################## +function deployCortxLocalBlockStorage() +{ + node_selector_list=$1 + node_name_list=$2 + namespace=$3 + printf "######################################################\n" + printf "# Deploy CORTX Local Block Storage \n" + printf "######################################################\n" + for i in "${!node_selector_list[@]}"; do + node_name=${node_name_list[i]} + node_selector=${node_selector_list[i]} + + storage_size_file_path="cortx-cloud-helm-pkg/cortx-data-provisioner/mnt-blk-storage-size-$node_name.txt" + storage_size=[] + size_count=0 + while IFS=' ' read -r size || [[ -n "$size" ]]; do + storage_size[size_count]=$size + size_count=$((size_count+1)) + done < "$storage_size_file_path" + + + file_path="cortx-cloud-helm-pkg/cortx-data-provisioner/mnt-blk-info-$node_name.txt" + count=001 + size_count=0 + while IFS=' ' read -r mount_path || [[ -n "$mount_path" ]]; do + mount_base_dir=$( echo "$mount_path" | sed -e 's/\/.*\///g') + count_str=$(printf "%03d" $count) + count=$((count+1)) + helm_name1="cortx-data-blk-data$count_str-$node_name" + storage_class_name1="local-blk-storage$count_str-$node_name" + pvc1_name="cortx-data-$mount_base_dir-pvc-$node_name" + pv1_name="cortx-data-$mount_base_dir-pv-$node_name" + helm install $helm_name1 cortx-cloud-helm-pkg/cortx-data-blk-data \ + --set cortxblkdata.nodename=$node_selector \ + --set cortxblkdata.storage.localpath=$mount_path \ + --set cortxblkdata.storage.size=${storage_size[size_count]} \ + --set cortxblkdata.storageclass=$storage_class_name1 \ + --set cortxblkdata.storage.pvc.name=$pvc1_name \ + --set cortxblkdata.storage.pv.name=$pv1_name \ + --set cortxblkdata.storage.volumemode="Block" \ + --set namespace=$namespace + size_count=$((size_count+1)) + done < "$file_path" + done +} -# Get the storage paths to use -local_storage=$(parseSolution 'solution.common.storage.local') -local_storage=$(echo $local_storage | cut -f2 -d'>') -shared_storage=$(parseSolution 'solution.common.storage.shared') -shared_storage=$(echo $shared_storage | cut -f2 -d'>') -log_storage=$(parseSolution 'solution.common.storage.log') -log_storage=$(echo $log_storage | cut -f2 -d'>') +function deployCortxGlusterFS() +{ + node_name_list=$1 + node_selector_list=$2 + gluster_vol=$3 + gluster_pv_name=$4 + gluster_pvc_name=$5 + gluster_etc_path=$6 + storage_prov_path=$7 + namespace=$8 + printf "########################################################\n" + printf "# Deploy CORTX GlusterFS \n" + printf "########################################################\n" + # Deploy GlusterFS + first_node_name=${node_name_list[0]} + first_node_selector=${node_selector_list[0]} + + image=$(parseSolution 'solution.images.gluster') + image=$(echo $image | cut -f2 -d'>') + gluster_size=$(parseSolution 'solution.common.glusterfs.size') + gluster_size=$(echo $gluster_size | cut -f2 -d'>') + + helm install "cortx-gluster-$first_node_name" cortx-cloud-helm-pkg/cortx-gluster \ + --set cortxgluster.name="gluster-$first_node_name" \ + --set cortxgluster.nodename=$first_node_selector \ + --set cortxgluster.service.name="cortx-gluster-svc-$first_node_name" \ + --set cortxgluster.image=$image \ + --set cortxgluster.storagesize=$gluster_size \ + --set cortxgluster.storageclass="cortx-gluster-storage" \ + --set cortxgluster.pv.path=$gluster_vol \ + --set cortxgluster.pv.name=$gluster_pv_name \ + --set cortxgluster.pvc.name=$gluster_pvc_name \ + --set cortxgluster.hostpath.etc=$gluster_etc_path \ + --set cortxgluster.hostpath.logs="$storage_prov_path/var/log/glusterfs" \ + --set cortxgluster.hostpath.config="$storage_prov_path/var/lib/glusterd" \ + --set namespace=$namespace + num_nodes=1 + + printf "\nWait for GlusterFS endpoint to be ready" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a service_status <<< "$line" + if [[ "${service_status[2]}" == "" ]]; then + break + fi + count=$((count+1)) + done <<< "$(kubectl get endpoints -A | grep 'gluster-')" -# GlusterFS -gluster_vol="myvol" -gluster_folder="/etc/gluster" -gluster_etc_path="$storage_prov_path/$gluster_folder" -gluster_pv_name="gluster-default-volume" -gluster_pvc_name="gluster-claim" + if [[ $num_nodes -eq $count ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n" + + printf "Wait for GlusterFS pod to be ready" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[2]}" + if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + break + fi + count=$((count+1)) + done <<< "$(kubectl get pods -A | grep 'gluster-')" -printf "######################################################\n" -printf "# Deploy CORTX Local Block Storage \n" -printf "######################################################\n" -for i in "${!node_selector_list[@]}"; do - node_name=${node_name_list[i]} - node_selector=${node_selector_list[i]} - - storage_size_file_path="cortx-cloud-helm-pkg/cortx-data-provisioner/mnt-blk-storage-size-$node_name.txt" - storage_size=[] - size_count=0 - while IFS=' ' read -r size || [[ -n "$size" ]]; do - storage_size[size_count]=$size - size_count=$((size_count+1)) - done < "$storage_size_file_path" - - - file_path="cortx-cloud-helm-pkg/cortx-data-provisioner/mnt-blk-info-$node_name.txt" - count=001 - size_count=0 - while IFS=' ' read -r mount_path || [[ -n "$mount_path" ]]; do - mount_base_dir=$( echo "$mount_path" | sed -e 's/\/.*\///g') - count_str=$(printf "%03d" $count) - count=$((count+1)) - helm_name1="cortx-data-blk-data$count_str-$node_name" - storage_class_name1="local-blk-storage$count_str-$node_name" - pvc1_name="cortx-data-$mount_base_dir-pvc-$node_name" - pv1_name="cortx-data-$mount_base_dir-pv-$node_name" - helm install $helm_name1 cortx-cloud-helm-pkg/cortx-data-blk-data \ - --set cortxblkdata.nodename=$node_selector \ - --set cortxblkdata.storage.localpath=$mount_path \ - --set cortxblkdata.storage.size=${storage_size[size_count]} \ - --set cortxblkdata.storageclass=$storage_class_name1 \ - --set cortxblkdata.storage.pvc.name=$pvc1_name \ - --set cortxblkdata.storage.pv.name=$pv1_name \ - --set cortxblkdata.storage.volumemode="Block" \ - --set namespace=$namespace - size_count=$((size_count+1)) - done < "$file_path" -done + if [[ $num_nodes -eq $count ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n\n" -printf "########################################################\n" -printf "# Deploy CORTX GlusterFS \n" -printf "########################################################\n" -# Deploy GlusterFS -first_node_name=${node_name_list[0]} -first_node_selector=${node_selector_list[0]} - -image=$(parseSolution 'solution.images.gluster') -image=$(echo $image | cut -f2 -d'>') -gluster_size=$(parseSolution 'solution.common.glusterfs.size') -gluster_size=$(echo $gluster_size | cut -f2 -d'>') - -helm install "cortx-gluster-$first_node_name" cortx-cloud-helm-pkg/cortx-gluster \ - --set cortxgluster.name="gluster-$first_node_name" \ - --set cortxgluster.nodename=$first_node_selector \ - --set cortxgluster.service.name="cortx-gluster-svc-$first_node_name" \ - --set cortxgluster.image=$image \ - --set cortxgluster.storagesize=$gluster_size \ - --set cortxgluster.storageclass="cortx-gluster-storage" \ - --set cortxgluster.pv.path=$gluster_vol \ - --set cortxgluster.pv.name=$gluster_pv_name \ - --set cortxgluster.pvc.name=$gluster_pvc_name \ - --set cortxgluster.hostpath.etc=$gluster_etc_path \ - --set cortxgluster.hostpath.logs="$storage_prov_path/var/log/glusterfs" \ - --set cortxgluster.hostpath.config="$storage_prov_path/var/lib/glusterd" \ - --set namespace=$namespace -num_nodes=1 - -printf "\nWait for GlusterFS endpoint to be ready" -while true; do + # Build Gluster endpoint array + gluster_ep_array=[] count=0 while IFS= read -r line; do - IFS=" " read -r -a service_status <<< "$line" - if [[ "${service_status[2]}" == "" ]]; then - break - fi + IFS=" " read -r -a my_array <<< "$line" + gluster_ep_array[count]=$line count=$((count+1)) - done <<< "$(kubectl get endpoints -A | grep 'gluster-')" + done <<< "$(kubectl get pods -A -o wide | grep 'gluster-')" - if [[ $num_nodes -eq $count ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n" - -printf "Wait for GlusterFS pod to be ready" -while true; do + gluster_and_host_name_arr=[] + # Loop through all gluster endpoint array and find endoint IP address + # and gluster node name count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[2]}" - if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then - break + first_gluster_node_name='' + first_gluster_ip='' + replica_list='' + for gluster_ep in "${gluster_ep_array[@]}" + do + IFS=" " read -r -a my_array <<< "$gluster_ep" + gluster_ep_ip=${my_array[6]} + gluster_node_name=${my_array[1]} + gluster_and_host_name_arr[count]="${gluster_ep_ip} ${gluster_node_name}" + if [[ "$count" == 0 ]]; then + first_gluster_node_name=$gluster_node_name + first_gluster_ip=$gluster_ep_ip + else + kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster peer probe $gluster_ep_ip fi + replica_list+="$gluster_ep_ip:$gluster_folder " count=$((count+1)) - done <<< "$(kubectl get pods -A | grep 'gluster-')" - - if [[ $num_nodes -eq $count ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" - -# Build Gluster endpoint array -gluster_ep_array=[] -count=0 -while IFS= read -r line; do - IFS=" " read -r -a my_array <<< "$line" - gluster_ep_array[count]=$line - count=$((count+1)) -done <<< "$(kubectl get pods -A -o wide | grep 'gluster-')" + done -gluster_and_host_name_arr=[] -# Loop through all gluster endpoint array and find endoint IP address -# and gluster node name -count=0 -first_gluster_node_name='' -first_gluster_ip='' -replica_list='' -for gluster_ep in "${gluster_ep_array[@]}" -do - IFS=" " read -r -a my_array <<< "$gluster_ep" - gluster_ep_ip=${my_array[6]} - gluster_node_name=${my_array[1]} - gluster_and_host_name_arr[count]="${gluster_ep_ip} ${gluster_node_name}" - if [[ "$count" == 0 ]]; then - first_gluster_node_name=$gluster_node_name - first_gluster_ip=$gluster_ep_ip + len_array=${#gluster_ep_array[@]} + if [[ ${#gluster_ep_array[@]} -ge 2 ]] + then + # Create replica gluster volumes + kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster volume create $gluster_vol replica $len_array $replica_list force else - kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster peer probe $gluster_ep_ip + # Add gluster volume + kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster volume create $gluster_vol $first_gluster_ip:$gluster_folder force fi - replica_list+="$gluster_ep_ip:$gluster_folder " - count=$((count+1)) -done -len_array=${#gluster_ep_array[@]} -if [[ ${#gluster_ep_array[@]} -ge 2 ]] -then - # Create replica gluster volumes - kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster volume create $gluster_vol replica $len_array $replica_list force -else - # Add gluster volume - kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster volume create $gluster_vol $first_gluster_ip:$gluster_folder force -fi + # Start gluster volume + echo y | kubectl exec -i $first_gluster_node_name --namespace=$namespace --namespace=$namespace -- gluster volume start $gluster_vol +} -# Start gluster volume -echo y | kubectl exec -i $first_gluster_node_name --namespace=$namespace --namespace=$namespace -- gluster volume start $gluster_vol - -printf "########################################################\n" -printf "# Deploy CORTX Configmap \n" -printf "########################################################\n" -# Delete all stale auto gen folders -rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-cfgmap -rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-control -rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-secret -rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/node-info -rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/storage-info -for i in "${!node_name_list[@]}"; do - rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-${node_name_list[i]} -done +function deleteStaleAutoGenFolders() +{ + node_name_list=$1 + # Delete all stale auto gen folders + rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-cfgmap + rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-control + rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-secret + rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/node-info + rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/storage-info + for i in "${!node_name_list[@]}"; do + rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-${node_name_list[i]} + done +} -# Default path to CORTX configmap -cfgmap_path="./cortx-cloud-helm-pkg/cortx-configmap" +function deployCortxConfigMap() +{ + cfgmap_path=$1 + node_name_list=$2 + cvg_index_list=$3 + namespace=$4 + printf "########################################################\n" + printf "# Deploy CORTX Configmap \n" + printf "########################################################\n" + # Create node template folder + node_info_folder="$cfgmap_path/node-info" + mkdir -p $node_info_folder + + # Create storage template folder + storage_info_folder="$cfgmap_path/storage-info" + mkdir -p $storage_info_folder + storage_info_temp_folder="$storage_info_folder/temp_folder" + mkdir -p $storage_info_temp_folder + + # Create auto-gen config folder + auto_gen_path="$cfgmap_path/auto-gen-cfgmap" + mkdir -p $auto_gen_path + + # Generate config files + for i in "${!node_name_list[@]}"; do + new_gen_file="$auto_gen_path/config.yaml" + cp "$cfgmap_path/templates/config-template.yaml" $new_gen_file + # 3rd party endpoints + kafka_endpoint="kafka.default.svc.cluster.local" + openldap_endpoint="openldap-svc.default.svc.cluster.local" + consul_endpoint="consul-server.default.svc.cluster.local" + openldap_servers="" + while IFS= read -r line; do + IFS=" " read -r -a my_array <<< "$line" + if [ "$openldap_servers" == "" ] + then + openldap_servers="- ""${my_array[1]}"".""$openldap_endpoint" + else + openldap_servers="$openldap_servers"$'\n'"- ""${my_array[1]}"".""$openldap_endpoint" + fi + done <<< "$(kubectl get pods -A | grep 'openldap-')" + + ./parse_scripts/subst.sh $new_gen_file "cortx.external.kafka.endpoints" $kafka_endpoint + ./parse_scripts/subst.sh $new_gen_file "cortx.external.openldap.endpoints" $openldap_endpoint + ./parse_scripts/yaml_insert_block.sh $new_gen_file "$openldap_servers" 8 "cortx.external.openldap.servers" + ./parse_scripts/subst.sh $new_gen_file "cortx.external.consul.endpoints" $consul_endpoint + ./parse_scripts/subst.sh $new_gen_file "cortx.io.svc" "cortx-io-svc" + ./parse_scripts/subst.sh $new_gen_file "cortx.num_s3_inst" $(extractBlock 'solution.common.s3.num_inst') + ./parse_scripts/subst.sh $new_gen_file "cortx.num_motr_inst" $(extractBlock 'solution.common.motr.num_client_inst') + ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.local" $(extractBlock 'solution.common.storage.local') + ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.shared" $(extractBlock 'solution.common.storage.shared') + ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.log" $(extractBlock 'solution.common.storage.log') + # Generate node file with type storage_node in "node-info" folder + new_gen_file="$node_info_folder/cluster-storage-node-${node_name_list[$i]}.yaml" + cp "$cfgmap_path/templates/cluster-node-template.yaml" $new_gen_file + ./parse_scripts/subst.sh $new_gen_file "cortx.node.name" "cortx-data-headless-svc-${node_name_list[$i]}" + uuid_str=$(UUID=$(uuidgen); echo ${UUID//-/}) + ./parse_scripts/subst.sh $new_gen_file "cortx.pod.uuid" "$uuid_str" + ./parse_scripts/subst.sh $new_gen_file "cortx.svc.name" "cortx-data-headless-svc-${node_name_list[$i]}" + ./parse_scripts/subst.sh $new_gen_file "cortx.node.type" "storage_node" + + # Create data machine id file + auto_gen_node_path="$cfgmap_path/auto-gen-${node_name_list[$i]}/data" + mkdir -p $auto_gen_node_path + echo $uuid_str > $auto_gen_node_path/id + done -# Create node template folder -node_info_folder="$cfgmap_path/node-info" -mkdir -p $node_info_folder - -# Create storage template folder -storage_info_folder="$cfgmap_path/storage-info" -mkdir -p $storage_info_folder -storage_info_temp_folder="$storage_info_folder/temp_folder" -mkdir -p $storage_info_temp_folder - -# Create auto-gen config folder -auto_gen_path="$cfgmap_path/auto-gen-cfgmap" -mkdir -p $auto_gen_path - -# Generate config files -for i in "${!node_name_list[@]}"; do - new_gen_file="$auto_gen_path/config.yaml" - cp "$cfgmap_path/templates/config-template.yaml" $new_gen_file - # 3rd party endpoints - kafka_endpoint="kafka.default.svc.cluster.local" - openldap_endpoint="openldap-svc.default.svc.cluster.local" - consul_endpoint="consul-server.default.svc.cluster.local" - openldap_servers="" - while IFS= read -r line; do - IFS=" " read -r -a my_array <<< "$line" - if [ "$openldap_servers" == "" ] - then - openldap_servers="- ""${my_array[1]}"".""$openldap_endpoint" - else - openldap_servers="$openldap_servers"$'\n'"- ""${my_array[1]}"".""$openldap_endpoint" - fi - done <<< "$(kubectl get pods -A | grep 'openldap-')" - - ./parse_scripts/subst.sh $new_gen_file "cortx.external.kafka.endpoints" $kafka_endpoint - ./parse_scripts/subst.sh $new_gen_file "cortx.external.openldap.endpoints" $openldap_endpoint - ./parse_scripts/yaml_insert_block.sh $new_gen_file "$openldap_servers" 8 "cortx.external.openldap.servers" - ./parse_scripts/subst.sh $new_gen_file "cortx.external.consul.endpoints" $consul_endpoint - ./parse_scripts/subst.sh $new_gen_file "cortx.io.svc" "cortx-io-svc" - ./parse_scripts/subst.sh $new_gen_file "cortx.num_s3_inst" $(extractBlock 'solution.common.s3.num_inst') - ./parse_scripts/subst.sh $new_gen_file "cortx.num_motr_inst" $(extractBlock 'solution.common.motr.num_client_inst') - ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.local" $(extractBlock 'solution.common.storage.local') - ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.shared" $(extractBlock 'solution.common.storage.shared') - ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.log" $(extractBlock 'solution.common.storage.log') - # Generate node file with type storage_node in "node-info" folder - new_gen_file="$node_info_folder/cluster-storage-node-${node_name_list[$i]}.yaml" + # Generate node file with type control_node in "node-info" folder + new_gen_file="$node_info_folder/cluster-control-node.yaml" cp "$cfgmap_path/templates/cluster-node-template.yaml" $new_gen_file - ./parse_scripts/subst.sh $new_gen_file "cortx.node.name" "cortx-data-headless-svc-${node_name_list[$i]}" + ./parse_scripts/subst.sh $new_gen_file "cortx.node.name" "cortx-control-headless-svc" uuid_str=$(UUID=$(uuidgen); echo ${UUID//-/}) ./parse_scripts/subst.sh $new_gen_file "cortx.pod.uuid" "$uuid_str" - ./parse_scripts/subst.sh $new_gen_file "cortx.svc.name" "cortx-data-headless-svc-${node_name_list[$i]}" - ./parse_scripts/subst.sh $new_gen_file "cortx.node.type" "storage_node" - - # Create data machine id file - auto_gen_node_path="$cfgmap_path/auto-gen-${node_name_list[$i]}/data" - mkdir -p $auto_gen_node_path - echo $uuid_str > $auto_gen_node_path/id -done - -# Generate node file with type control_node in "node-info" folder -new_gen_file="$node_info_folder/cluster-control-node.yaml" -cp "$cfgmap_path/templates/cluster-node-template.yaml" $new_gen_file -./parse_scripts/subst.sh $new_gen_file "cortx.node.name" "cortx-control-headless-svc" -uuid_str=$(UUID=$(uuidgen); echo ${UUID//-/}) -./parse_scripts/subst.sh $new_gen_file "cortx.pod.uuid" "$uuid_str" -./parse_scripts/subst.sh $new_gen_file "cortx.svc.name" "cortx-control-headless-svc" -./parse_scripts/subst.sh $new_gen_file "cortx.node.type" "control_node" - -# Create control machine id file -auto_gen_control_path="$cfgmap_path/auto-gen-control" -mkdir -p $auto_gen_control_path -echo $uuid_str > $auto_gen_control_path/id - -# Copy cluster template -cp "$cfgmap_path/templates/cluster-template.yaml" "$auto_gen_path/cluster.yaml" - -# Insert all node info stored in "node-info" folder into "cluster.yaml" file -cluster_uuid=$(UUID=$(uuidgen); echo ${UUID//-/}) -extract_output="" -node_info_folder="$cfgmap_path/node-info" -./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cortx.cluster.id" $cluster_uuid - -# Populate the storage set info -storage_set_name=$(parseSolution 'solution.common.storage_sets.name') -storage_set_name=$(echo $storage_set_name | cut -f2 -d'>') -storage_set_dur_sns=$(parseSolution 'solution.common.storage_sets.durability.sns') -storage_set_dur_sns=$(echo $storage_set_dur_sns | cut -f2 -d'>') -storage_set_dur_dix=$(parseSolution 'solution.common.storage_sets.durability.dix') -storage_set_dur_dix=$(echo $storage_set_dur_dix | cut -f2 -d'>') - -./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cluster.storage_sets.name" $storage_set_name -./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cluster.storage_sets.durability.sns" $storage_set_dur_sns -./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cluster.storage_sets.durability.dix" $storage_set_dur_dix - -for fname in ./cortx-cloud-helm-pkg/cortx-configmap/node-info/*; do - if [ "$extract_output" == "" ] - then - extract_output="$(./parse_scripts/yaml_extract_block.sh $fname)" - else - extract_output="$extract_output"$'\n'"$(./parse_scripts/yaml_extract_block.sh $fname)" - fi -done -./parse_scripts/yaml_insert_block.sh "$auto_gen_path/cluster.yaml" "$extract_output" 4 "cluster.storage_sets.nodes" + ./parse_scripts/subst.sh $new_gen_file "cortx.svc.name" "cortx-control-headless-svc" + ./parse_scripts/subst.sh $new_gen_file "cortx.node.type" "control_node" + + # Create control machine id file + auto_gen_control_path="$cfgmap_path/auto-gen-control" + mkdir -p $auto_gen_control_path + echo $uuid_str > $auto_gen_control_path/id + + # Copy cluster template + cp "$cfgmap_path/templates/cluster-template.yaml" "$auto_gen_path/cluster.yaml" + + # Insert all node info stored in "node-info" folder into "cluster.yaml" file + cluster_uuid=$(UUID=$(uuidgen); echo ${UUID//-/}) + extract_output="" + node_info_folder="$cfgmap_path/node-info" + ./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cortx.cluster.id" $cluster_uuid + + # Populate the storage set info + storage_set_name=$(parseSolution 'solution.common.storage_sets.name') + storage_set_name=$(echo $storage_set_name | cut -f2 -d'>') + storage_set_dur_sns=$(parseSolution 'solution.common.storage_sets.durability.sns') + storage_set_dur_sns=$(echo $storage_set_dur_sns | cut -f2 -d'>') + storage_set_dur_dix=$(parseSolution 'solution.common.storage_sets.durability.dix') + storage_set_dur_dix=$(echo $storage_set_dur_dix | cut -f2 -d'>') + + ./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cluster.storage_sets.name" $storage_set_name + ./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cluster.storage_sets.durability.sns" $storage_set_dur_sns + ./parse_scripts/subst.sh "$auto_gen_path/cluster.yaml" "cluster.storage_sets.durability.dix" $storage_set_dur_dix + + for fname in ./cortx-cloud-helm-pkg/cortx-configmap/node-info/*; do + if [ "$extract_output" == "" ] + then + extract_output="$(./parse_scripts/yaml_extract_block.sh $fname)" + else + extract_output="$extract_output"$'\n'"$(./parse_scripts/yaml_extract_block.sh $fname)" + fi + done + ./parse_scripts/yaml_insert_block.sh "$auto_gen_path/cluster.yaml" "$extract_output" 4 "cluster.storage_sets.nodes" + + for cvg_index in "${cvg_index_list[@]}"; do + storage_cvg_data_auto_gen_file="$storage_info_temp_folder/cluster-storage-$cvg_index-data.yaml" + filter="solution.storage.$cvg_index.devices.data.d*.device" + cvg_devices_output=$(parseSolution $filter) + IFS=';' read -r -a cvg_dev_var_val_array <<< "$cvg_devices_output" + for cvg_dev_var_val_element in "${cvg_dev_var_val_array[@]}"; do + cvg_dev=$(echo $cvg_dev_var_val_element | cut -f2 -d'>') + echo "- $cvg_dev" >> $storage_cvg_data_auto_gen_file + done + + # Substitute all the variables in the template file + storage_info_gen_file="$storage_info_folder/cluster-storage-$cvg_index-info.yaml" + cp "$cfgmap_path/templates/cluster-storage-template.yaml" $storage_info_gen_file + + cvg_name_output=$(parseSolution "solution.storage.$cvg_index.name") + cvg_name=$(echo $cvg_name_output | cut -f2 -d'>') + ./parse_scripts/subst.sh $storage_info_gen_file "cortx.storage.name" $cvg_name + + cvg_type_output=$(parseSolution "solution.storage.$cvg_index.type") + cvg_type=$(echo $cvg_type_output | cut -f2 -d'>') + ./parse_scripts/subst.sh $storage_info_gen_file "cortx.storage.type" $cvg_type + + cvg_metadata_output=$(parseSolution "solution.storage.$cvg_index.devices.metadata.device") + cvg_metadata=$(echo $cvg_metadata_output | cut -f2 -d'>') + ./parse_scripts/subst.sh $storage_info_gen_file "cortx.metadata.dev_partition" $cvg_metadata + + extract_output="$(./parse_scripts/yaml_extract_block.sh $storage_cvg_data_auto_gen_file)" + ./parse_scripts/yaml_insert_block.sh "$storage_info_gen_file" "$extract_output" 4 "cortx.data.dev_partition" + done + # Remove "storage-info/temp_folder" + rm -rf $storage_info_temp_folder + # Insert data device info stored in 'storage-info' folder into 'cluster-storage-node.yaml' file + extract_output="" + for fname in ./cortx-cloud-helm-pkg/cortx-configmap/storage-info/*; do + if [ "$extract_output" == "" ] + then + extract_output="$(./parse_scripts/yaml_extract_block.sh $fname)" + else + extract_output="$extract_output"$'\n'"$(./parse_scripts/yaml_extract_block.sh $fname)" + fi + done + ./parse_scripts/yaml_insert_block.sh "$auto_gen_path/cluster.yaml" "$extract_output" 4 "cluster.storage_list" + # Delete node-info folder + node_info_folder="$cfgmap_path/node-info" + rm -rf $node_info_folder -cvg_output=$(parseSolution 'solution.storage.cvg*.name') -IFS=';' read -r -a cvg_var_val_array <<< "$cvg_output" -# Build CVG index list (ex: [cvg1, cvg2, cvg3]) -cvg_index_list=[] -count=0 -for cvg_var_val_element in "${cvg_var_val_array[@]}"; do - cvg_name=$(echo $cvg_var_val_element | cut -f2 -d'>') - cvg_filter=$(echo $cvg_var_val_element | cut -f1 -d'>') - cvg_index=$(echo $cvg_filter | cut -f3 -d'.') - cvg_index_list[$count]=$cvg_index - count=$((count+1)) -done + # Create config maps + auto_gen_path="$cfgmap_path/auto-gen-cfgmap" + kubectl_cmd_output=$(kubectl create configmap "cortx-cfgmap" \ + --namespace=$namespace \ + --from-file=$auto_gen_path) + if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then + printf "Exit early. Create config map 'cortx-cfgmap' failed with error:\n$kubectl_cmd_output\n" + exit 1 + fi + echo $kubectl_cmd_output -for cvg_index in "${cvg_index_list[@]}"; do - storage_cvg_data_auto_gen_file="$storage_info_temp_folder/cluster-storage-$cvg_index-data.yaml" - filter="solution.storage.$cvg_index.devices.data.d*.device" - cvg_devices_output=$(parseSolution $filter) - IFS=';' read -r -a cvg_dev_var_val_array <<< "$cvg_devices_output" - for cvg_dev_var_val_element in "${cvg_dev_var_val_array[@]}"; do - cvg_dev=$(echo $cvg_dev_var_val_element | cut -f2 -d'>') - echo "- $cvg_dev" >> $storage_cvg_data_auto_gen_file + # Create data machine ID config maps + for i in "${!node_name_list[@]}"; do + auto_gen_cfgmap_path="$cfgmap_path/auto-gen-${node_name_list[i]}/data" + kubectl_cmd_output=$(kubectl create configmap "cortx-data-machine-id-cfgmap-${node_name_list[i]}" \ + --namespace=$namespace \ + --from-file=$auto_gen_cfgmap_path) + if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then + printf "Exit early. Create config map 'cortx-data-machine-id-cfgmap-${node_name_list[i]}' failed with error:\n$kubectl_cmd_output\n" + exit 1 + fi done - - # Substitute all the variables in the template file - storage_info_gen_file="$storage_info_folder/cluster-storage-$cvg_index-info.yaml" - cp "$cfgmap_path/templates/cluster-storage-template.yaml" $storage_info_gen_file - - cvg_name_output=$(parseSolution "solution.storage.$cvg_index.name") - cvg_name=$(echo $cvg_name_output | cut -f2 -d'>') - ./parse_scripts/subst.sh $storage_info_gen_file "cortx.storage.name" $cvg_name - - cvg_type_output=$(parseSolution "solution.storage.$cvg_index.type") - cvg_type=$(echo $cvg_type_output | cut -f2 -d'>') - ./parse_scripts/subst.sh $storage_info_gen_file "cortx.storage.type" $cvg_type - - cvg_metadata_output=$(parseSolution "solution.storage.$cvg_index.devices.metadata.device") - cvg_metadata=$(echo $cvg_metadata_output | cut -f2 -d'>') - ./parse_scripts/subst.sh $storage_info_gen_file "cortx.metadata.dev_partition" $cvg_metadata - - extract_output="$(./parse_scripts/yaml_extract_block.sh $storage_cvg_data_auto_gen_file)" - ./parse_scripts/yaml_insert_block.sh "$storage_info_gen_file" "$extract_output" 4 "cortx.data.dev_partition" -done -# Remove "storage-info/temp_folder" -rm -rf $storage_info_temp_folder -# Insert data device info stored in 'storage-info' folder into 'cluster-storage-node.yaml' file -extract_output="" -for fname in ./cortx-cloud-helm-pkg/cortx-configmap/storage-info/*; do - if [ "$extract_output" == "" ] - then - extract_output="$(./parse_scripts/yaml_extract_block.sh $fname)" - else - extract_output="$extract_output"$'\n'"$(./parse_scripts/yaml_extract_block.sh $fname)" - fi -done -./parse_scripts/yaml_insert_block.sh "$auto_gen_path/cluster.yaml" "$extract_output" 4 "cluster.storage_list" - -# Delete node-info folder -node_info_folder="$cfgmap_path/node-info" -rm -rf $node_info_folder - -# Create config maps -auto_gen_path="$cfgmap_path/auto-gen-cfgmap" -kubectl_cmd_output=$(kubectl create configmap "cortx-cfgmap" \ - --namespace=$namespace \ - --from-file=$auto_gen_path) -if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then - printf "Exit early. Create config map 'cortx-cfgmap' failed with error:\n$kubectl_cmd_output\n" - exit 1 -fi -echo $kubectl_cmd_output + echo $kubectl_cmd_output -# Create data machine ID config maps -for i in "${!node_name_list[@]}"; do - auto_gen_cfgmap_path="$cfgmap_path/auto-gen-${node_name_list[i]}/data" - kubectl_cmd_output=$(kubectl create configmap "cortx-data-machine-id-cfgmap-${node_name_list[i]}" \ + # Create control machine ID config maps + auto_gen_control_path="$cfgmap_path/auto-gen-control" + kubectl_cmd_output=$(kubectl create configmap "cortx-control-machine-id-cfgmap" \ --namespace=$namespace \ - --from-file=$auto_gen_cfgmap_path) + --from-file=$auto_gen_control_path) if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then - printf "Exit early. Create config map 'cortx-data-machine-id-cfgmap-${node_name_list[i]}' failed with error:\n$kubectl_cmd_output\n" - exit 1 - fi -done -echo $kubectl_cmd_output - -# Create control machine ID config maps -auto_gen_control_path="$cfgmap_path/auto-gen-control" -kubectl_cmd_output=$(kubectl create configmap "cortx-control-machine-id-cfgmap" \ - --namespace=$namespace \ - --from-file=$auto_gen_control_path) -if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then - printf "Exit early. Create config map 'cortx-control-machine-id-cfgmap' failed with error:\n$kubectl_cmd_output\n" - exit 1 -fi -echo $kubectl_cmd_output - -# Create SSL cert config map -ssl_cert_path="$cfgmap_path/ssl-cert" -kubectl_cmd_output=$(kubectl create configmap "cortx-ssl-cert-cfgmap" \ - --namespace=$namespace \ - --from-file=$ssl_cert_path) -if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then - printf "Exit early. Create config map 'cortx-ssl-cert-cfgmap' failed with error:\n$kubectl_cmd_output\n" - exit 1 -fi -echo $kubectl_cmd_output - -printf "########################################################\n" -printf "# Deploy CORTX Secrets \n" -printf "########################################################\n" -# Parse secret from the solution file and create all secret yaml files -# in the "auto-gen-secret" folder -secret_auto_gen_path="$cfgmap_path/auto-gen-secret" -mkdir -p $secret_auto_gen_path -output=$(./parse_scripts/parse_yaml.sh $solution_yaml "solution.secrets.name") -IFS=';' read -r -a parsed_secret_name_array <<< "$output" -for secret_name in "${parsed_secret_name_array[@]}" -do - secret_fname=$(echo $secret_name | cut -f2 -d'>') - yaml_content_path=$(echo $secret_name | cut -f1 -d'>') - yaml_content_path=${yaml_content_path/.name/".content"} - secrets="$(./parse_scripts/yaml_extract_block.sh $solution_yaml $yaml_content_path 2)" - - new_secret_gen_file="$secret_auto_gen_path/$secret_fname.yaml" - cp "$cfgmap_path/templates/secret-template.yaml" $new_secret_gen_file - ./parse_scripts/subst.sh $new_secret_gen_file "secret.name" "$secret_fname" - ./parse_scripts/subst.sh $new_secret_gen_file "secret.content" "$secrets" - - kubectl_cmd_output=$(kubectl create -f $new_secret_gen_file --namespace=$namespace 2>&1) - - if [[ "$kubectl_cmd_output" == *"BadRequest"* ]]; then - printf "Exit early. Create secret failed with error:\n$kubectl_cmd_output\n" + printf "Exit early. Create config map 'cortx-control-machine-id-cfgmap' failed with error:\n$kubectl_cmd_output\n" exit 1 fi echo $kubectl_cmd_output - control_prov_secret_path="./cortx-cloud-helm-pkg/cortx-control-provisioner/secret-info.txt" - control_secret_path="./cortx-cloud-helm-pkg/cortx-control/secret-info.txt" - data_prov_secret_path="./cortx-cloud-helm-pkg/cortx-data-provisioner/secret-info.txt" - data_secret_path="./cortx-cloud-helm-pkg/cortx-data/secret-info.txt" - if [[ -s $control_prov_secret_path ]]; then - printf "\n" >> $control_prov_secret_path - fi - if [[ -s $control_secret_path ]]; then - printf "\n" >> $control_secret_path - fi - if [[ -s $data_prov_secret_path ]]; then - printf "\n" >> $data_prov_secret_path - fi - if [[ -s $data_secret_path ]]; then - printf "\n" >> $data_secret_path + # Create SSL cert config map + ssl_cert_path="$cfgmap_path/ssl-cert" + kubectl_cmd_output=$(kubectl create configmap "cortx-ssl-cert-cfgmap" \ + --namespace=$namespace \ + --from-file=$ssl_cert_path) + if [[ "$kubectl_cmd_output" == *"no such file or directory"* ]]; then + printf "Exit early. Create config map 'cortx-ssl-cert-cfgmap' failed with error:\n$kubectl_cmd_output\n" + exit 1 fi - printf "$secret_fname" >> $control_prov_secret_path - printf "$secret_fname" >> $control_secret_path - printf "$secret_fname" >> $data_prov_secret_path - printf "$secret_fname" >> $data_secret_path -done - + echo $kubectl_cmd_output +} -printf "########################################################\n" -printf "# Deploy CORTX Control Provisioner \n" -printf "########################################################\n" -cortxcontrolprov_image=$(parseSolution 'solution.images.cortxcontrolprov') -cortxcontrolprov_image=$(echo $cortxcontrolprov_image | cut -f2 -d'>') - -helm install "cortx-control-provisioner" cortx-cloud-helm-pkg/cortx-control-provisioner \ - --set cortxcontrolprov.name="cortx-control-provisioner-pod" \ - --set cortxcontrolprov.image=$cortxcontrolprov_image \ - --set cortxcontrolprov.service.clusterip.name="cortx-control-clusterip-svc" \ - --set cortxcontrolprov.service.headless.name="cortx-control-headless-svc" \ - --set cortxgluster.pv.name=$gluster_pv_name \ - --set cortxgluster.pv.mountpath=$shared_storage \ - --set cortxgluster.pvc.name=$gluster_pvc_name \ - --set cortxcontrolprov.cfgmap.name="cortx-cfgmap" \ - --set cortxcontrolprov.cfgmap.volmountname="config001" \ - --set cortxcontrolprov.cfgmap.mountpath="/etc/cortx/solution" \ - --set cortxcontrolprov.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ - --set cortxcontrolprov.sslcfgmap.volmountname="ssl-config001" \ - --set cortxcontrolprov.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ - --set cortxcontrolprov.machineid.name="cortx-control-machine-id-cfgmap" \ - --set cortxcontrolprov.localpathpvc.name="cortx-control-fs-local-pvc" \ - --set cortxcontrolprov.localpathpvc.mountpath="$local_storage" \ - --set cortxcontrolprov.localpathpvc.requeststoragesize="1Gi" \ - --set cortxcontrolprov.secretinfo="secret-info.txt" \ - --set namespace=$namespace - -# Check if all Cortx Control Provisioner is up and running -node_count=1 -printf "\nWait for CORTX Control Provisioner to complete" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - if [[ "${pod_status[2]}" != "Completed" ]]; then - if [[ "${pod_status[2]}" == "Error" ]]; then - printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" - exit 1 - fi - break +function deployCortxSecrets() +{ + cfgmap_path=$1 + solution_yaml=$2 + namespace=$3 + printf "########################################################\n" + printf "# Deploy CORTX Secrets \n" + printf "########################################################\n" + # Parse secret from the solution file and create all secret yaml files + # in the "auto-gen-secret" folder + secret_auto_gen_path="$cfgmap_path/auto-gen-secret" + mkdir -p $secret_auto_gen_path + output=$(./parse_scripts/parse_yaml.sh $solution_yaml "solution.secrets.name") + IFS=';' read -r -a parsed_secret_name_array <<< "$output" + for secret_name in "${parsed_secret_name_array[@]}" + do + secret_fname=$(echo $secret_name | cut -f2 -d'>') + yaml_content_path=$(echo $secret_name | cut -f1 -d'>') + yaml_content_path=${yaml_content_path/.name/".content"} + secrets="$(./parse_scripts/yaml_extract_block.sh $solution_yaml $yaml_content_path 2)" + + new_secret_gen_file="$secret_auto_gen_path/$secret_fname.yaml" + cp "$cfgmap_path/templates/secret-template.yaml" $new_secret_gen_file + ./parse_scripts/subst.sh $new_secret_gen_file "secret.name" "$secret_fname" + ./parse_scripts/subst.sh $new_secret_gen_file "secret.content" "$secrets" + + kubectl_cmd_output=$(kubectl create -f $new_secret_gen_file --namespace=$namespace 2>&1) + + if [[ "$kubectl_cmd_output" == *"BadRequest"* ]]; then + printf "Exit early. Create secret failed with error:\n$kubectl_cmd_output\n" + exit 1 fi - count=$((count+1)) - done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-provisioner-pod')" + echo $kubectl_cmd_output + + control_prov_secret_path="./cortx-cloud-helm-pkg/cortx-control-provisioner/secret-info.txt" + control_secret_path="./cortx-cloud-helm-pkg/cortx-control/secret-info.txt" + data_prov_secret_path="./cortx-cloud-helm-pkg/cortx-data-provisioner/secret-info.txt" + data_secret_path="./cortx-cloud-helm-pkg/cortx-data/secret-info.txt" + if [[ -s $control_prov_secret_path ]]; then + printf "\n" >> $control_prov_secret_path + fi + if [[ -s $control_secret_path ]]; then + printf "\n" >> $control_secret_path + fi + if [[ -s $data_prov_secret_path ]]; then + printf "\n" >> $data_prov_secret_path + fi + if [[ -s $data_secret_path ]]; then + printf "\n" >> $data_secret_path + fi + printf "$secret_fname" >> $control_prov_secret_path + printf "$secret_fname" >> $control_secret_path + printf "$secret_fname" >> $data_prov_secret_path + printf "$secret_fname" >> $data_secret_path + done +} - if [[ $node_count -eq $count ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" - -# Delete CORTX Provisioner Services -kubectl delete service "cortx-control-clusterip-svc" --namespace=$namespace -kubectl delete service "cortx-control-headless-svc" --namespace=$namespace - -printf "########################################################\n" -printf "# Deploy CORTX Data Provisioner \n" -printf "########################################################\n" -cortxdataprov_image=$(parseSolution 'solution.images.cortxdataprov') -cortxdataprov_image=$(echo $cortxdataprov_image | cut -f2 -d'>') - -for i in "${!node_selector_list[@]}"; do - node_name=${node_name_list[i]} - node_selector=${node_selector_list[i]} - helm install "cortx-data-provisioner-$node_name" cortx-cloud-helm-pkg/cortx-data-provisioner \ - --set cortxdataprov.name="cortx-data-provisioner-pod-$node_name" \ - --set cortxdataprov.image=$cortxdataprov_image \ - --set cortxdataprov.nodename=$node_name \ - --set cortxdataprov.mountblkinfo="mnt-blk-info-$node_name.txt" \ - --set cortxdataprov.service.clusterip.name="cortx-data-clusterip-svc-$node_name" \ - --set cortxdataprov.service.headless.name="cortx-data-headless-svc-$node_name" \ +function deployCortxControlProvisioner() +{ + gluster_pv_name=$1 + gluster_pvc_name=$2 + shared_storage=$3 + local_storage=$4 + namespace=$5 + printf "########################################################\n" + printf "# Deploy CORTX Control Provisioner \n" + printf "########################################################\n" + cortxcontrolprov_image=$(parseSolution 'solution.images.cortxcontrolprov') + cortxcontrolprov_image=$(echo $cortxcontrolprov_image | cut -f2 -d'>') + + helm install "cortx-control-provisioner" cortx-cloud-helm-pkg/cortx-control-provisioner \ + --set cortxcontrolprov.name="cortx-control-provisioner-pod" \ + --set cortxcontrolprov.image=$cortxcontrolprov_image \ + --set cortxcontrolprov.service.clusterip.name="cortx-control-clusterip-svc" \ + --set cortxcontrolprov.service.headless.name="cortx-control-headless-svc" \ --set cortxgluster.pv.name=$gluster_pv_name \ --set cortxgluster.pv.mountpath=$shared_storage \ --set cortxgluster.pvc.name=$gluster_pvc_name \ - --set cortxdataprov.cfgmap.name="cortx-cfgmap" \ - --set cortxdataprov.cfgmap.volmountname="config001-$node_name" \ - --set cortxdataprov.cfgmap.mountpath="/etc/cortx/solution" \ - --set cortxdataprov.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ - --set cortxdataprov.sslcfgmap.volmountname="ssl-config001" \ - --set cortxdataprov.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ - --set cortxdataprov.machineid.name="cortx-data-machine-id-cfgmap-$node_name" \ - --set cortxdataprov.localpathpvc.name="cortx-data-fs-local-pvc-$node_name" \ - --set cortxdataprov.localpathpvc.mountpath="$local_storage" \ - --set cortxdataprov.localpathpvc.requeststoragesize="1Gi" \ - --set cortxdataprov.secretinfo="secret-info.txt" \ + --set cortxcontrolprov.cfgmap.name="cortx-cfgmap" \ + --set cortxcontrolprov.cfgmap.volmountname="config001" \ + --set cortxcontrolprov.cfgmap.mountpath="/etc/cortx/solution" \ + --set cortxcontrolprov.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ + --set cortxcontrolprov.sslcfgmap.volmountname="ssl-config001" \ + --set cortxcontrolprov.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ + --set cortxcontrolprov.machineid.name="cortx-control-machine-id-cfgmap" \ + --set cortxcontrolprov.localpathpvc.name="cortx-control-fs-local-pvc" \ + --set cortxcontrolprov.localpathpvc.mountpath="$local_storage" \ + --set cortxcontrolprov.localpathpvc.requeststoragesize="1Gi" \ + --set cortxcontrolprov.secretinfo="secret-info.txt" \ --set namespace=$namespace -done - -# Check if all OpenLDAP are up and running -node_count="${#node_selector_list[@]}" -printf "\nWait for CORTX Data Provisioner to complete" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - if [[ "${pod_status[2]}" != "Completed" ]]; then - if [[ "${pod_status[2]}" == "Error" ]]; then - printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" - exit 1 + # Check if all Cortx Control Provisioner is up and running + node_count=1 + printf "\nWait for CORTX Control Provisioner to complete" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + if [[ "${pod_status[2]}" != "Completed" ]]; then + if [[ "${pod_status[2]}" == "Error" ]]; then + printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" + exit 1 + fi + break fi + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-provisioner-pod')" + + if [[ $node_count -eq $count ]]; then break + else + printf "." fi - count=$((count+1)) - done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-provisioner-pod-')" + sleep 1s + done + printf "\n\n" - if [[ $node_count -eq $count ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" - -# Delete CORTX Provisioner Services -for i in "${!node_selector_list[@]}"; do - node_name=${node_name_list[i]} - node_selector=${node_selector_list[i]} - num_nodes=$((num_nodes+1)) - kubectl delete service "cortx-data-clusterip-svc-$node_name" --namespace=$namespace - kubectl delete service "cortx-data-headless-svc-$node_name" --namespace=$namespace -done + # Delete CORTX Provisioner Services + kubectl delete service "cortx-control-clusterip-svc" --namespace=$namespace + kubectl delete service "cortx-control-headless-svc" --namespace=$namespace +} -printf "########################################################\n" -printf "# Deploy CORTX Control \n" -printf "########################################################\n" -cortxcontrol_image=$(parseSolution 'solution.images.cortxcontrol') -cortxcontrol_image=$(echo $cortxcontrol_image | cut -f2 -d'>') - -num_nodes=1 -# This local path pvc has to match with the one created by CORTX Control Provisioner -helm install "cortx-control" cortx-cloud-helm-pkg/cortx-control \ - --set cortxcontrol.name="cortx-control-pod" \ - --set cortxcontrol.image=$cortxcontrol_image \ - --set cortxcontrol.service.clusterip.name="cortx-control-clusterip-svc" \ - --set cortxcontrol.service.headless.name="cortx-control-headless-svc" \ - --set cortxcontrol.loadbal.name="cortx-control-loadbal-svc" \ - --set cortxcontrol.cfgmap.mountpath="/etc/cortx/solution" \ - --set cortxcontrol.cfgmap.name="cortx-cfgmap" \ - --set cortxcontrol.cfgmap.volmountname="config001" \ - --set cortxcontrol.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ - --set cortxcontrol.sslcfgmap.volmountname="ssl-config001" \ - --set cortxcontrol.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ - --set cortxcontrol.machineid.name="cortx-control-machine-id-cfgmap" \ - --set cortxcontrol.localpathpvc.name="cortx-control-fs-local-pvc" \ - --set cortxcontrol.localpathpvc.mountpath="$local_storage" \ - --set cortxcontrol.secretinfo="secret-info.txt" \ - --set cortxgluster.pv.name="gluster-default-name" \ - --set cortxgluster.pv.mountpath=$shared_storage \ - --set cortxgluster.pvc.name="gluster-claim" \ - --set namespace=$namespace - -printf "\nWait for CORTX Control to be ready" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[1]}" - if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then - if [[ "${pod_status[2]}" == "Error" ]]; then - printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" - exit 1 +function deployCortxDataProvisioner() +{ + node_selector_list=$1 + node_name_list=$2 + gluster_pv_name=$3 + gluster_pvc_name=$4 + shared_storage=$5 + local_storage=$6 + namespace=$7 + printf "########################################################\n" + printf "# Deploy CORTX Data Provisioner \n" + printf "########################################################\n" + cortxdataprov_image=$(parseSolution 'solution.images.cortxdataprov') + cortxdataprov_image=$(echo $cortxdataprov_image | cut -f2 -d'>') + + for i in "${!node_selector_list[@]}"; do + node_name=${node_name_list[i]} + node_selector=${node_selector_list[i]} + helm install "cortx-data-provisioner-$node_name" cortx-cloud-helm-pkg/cortx-data-provisioner \ + --set cortxdataprov.name="cortx-data-provisioner-pod-$node_name" \ + --set cortxdataprov.image=$cortxdataprov_image \ + --set cortxdataprov.nodename=$node_name \ + --set cortxdataprov.mountblkinfo="mnt-blk-info-$node_name.txt" \ + --set cortxdataprov.service.clusterip.name="cortx-data-clusterip-svc-$node_name" \ + --set cortxdataprov.service.headless.name="cortx-data-headless-svc-$node_name" \ + --set cortxgluster.pv.name=$gluster_pv_name \ + --set cortxgluster.pv.mountpath=$shared_storage \ + --set cortxgluster.pvc.name=$gluster_pvc_name \ + --set cortxdataprov.cfgmap.name="cortx-cfgmap" \ + --set cortxdataprov.cfgmap.volmountname="config001-$node_name" \ + --set cortxdataprov.cfgmap.mountpath="/etc/cortx/solution" \ + --set cortxdataprov.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ + --set cortxdataprov.sslcfgmap.volmountname="ssl-config001" \ + --set cortxdataprov.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ + --set cortxdataprov.machineid.name="cortx-data-machine-id-cfgmap-$node_name" \ + --set cortxdataprov.localpathpvc.name="cortx-data-fs-local-pvc-$node_name" \ + --set cortxdataprov.localpathpvc.mountpath="$local_storage" \ + --set cortxdataprov.localpathpvc.requeststoragesize="1Gi" \ + --set cortxdataprov.secretinfo="secret-info.txt" \ + --set namespace=$namespace + done + + # Check if all OpenLDAP are up and running + node_count="${#node_selector_list[@]}" + + printf "\nWait for CORTX Data Provisioner to complete" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + if [[ "${pod_status[2]}" != "Completed" ]]; then + if [[ "${pod_status[2]}" == "Error" ]]; then + printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" + exit 1 + fi + break fi + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-provisioner-pod-')" + + if [[ $node_count -eq $count ]]; then break + else + printf "." fi - count=$((count+1)) - done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-pod-')" + sleep 1s + done + printf "\n\n" + + # Delete CORTX Provisioner Services + for i in "${!node_selector_list[@]}"; do + node_name=${node_name_list[i]} + node_selector=${node_selector_list[i]} + num_nodes=$((num_nodes+1)) + kubectl delete service "cortx-data-clusterip-svc-$node_name" --namespace=$namespace + kubectl delete service "cortx-data-headless-svc-$node_name" --namespace=$namespace + done +} - if [[ $num_nodes -eq $count ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" - -printf "########################################################\n" -printf "# Deploy CORTX Data \n" -printf "########################################################\n" -cortxdata_image=$(parseSolution 'solution.images.cortxdata') -cortxdata_image=$(echo $cortxdata_image | cut -f2 -d'>') - -num_nodes=0 -for i in "${!node_selector_list[@]}"; do - num_nodes=$((num_nodes+1)) - node_name=${node_name_list[i]} - node_selector=${node_selector_list[i]} - helm install "cortx-data-$node_name" cortx-cloud-helm-pkg/cortx-data \ - --set cortxdata.name="cortx-data-pod-$node_name" \ - --set cortxdata.image=$cortxdata_image \ - --set cortxdata.nodename=$node_name \ - --set cortxdata.mountblkinfo="mnt-blk-info-$node_name.txt" \ - --set cortxdata.service.clusterip.name="cortx-data-clusterip-svc-$node_name" \ - --set cortxdata.service.headless.name="cortx-data-headless-svc-$node_name" \ - --set cortxdata.service.loadbal.name="cortx-data-loadbal-svc-$node_name" \ - --set cortxgluster.pv.name=$gluster_pv_name \ +function deployCortxControl() +{ + local_storage=$1 + shared_storage=$2 + namespace=$3 + printf "########################################################\n" + printf "# Deploy CORTX Control \n" + printf "########################################################\n" + cortxcontrol_image=$(parseSolution 'solution.images.cortxcontrol') + cortxcontrol_image=$(echo $cortxcontrol_image | cut -f2 -d'>') + + num_nodes=1 + # This local path pvc has to match with the one created by CORTX Control Provisioner + helm install "cortx-control" cortx-cloud-helm-pkg/cortx-control \ + --set cortxcontrol.name="cortx-control-pod" \ + --set cortxcontrol.image=$cortxcontrol_image \ + --set cortxcontrol.service.clusterip.name="cortx-control-clusterip-svc" \ + --set cortxcontrol.service.headless.name="cortx-control-headless-svc" \ + --set cortxcontrol.loadbal.name="cortx-control-loadbal-svc" \ + --set cortxcontrol.cfgmap.mountpath="/etc/cortx/solution" \ + --set cortxcontrol.cfgmap.name="cortx-cfgmap" \ + --set cortxcontrol.cfgmap.volmountname="config001" \ + --set cortxcontrol.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ + --set cortxcontrol.sslcfgmap.volmountname="ssl-config001" \ + --set cortxcontrol.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ + --set cortxcontrol.machineid.name="cortx-control-machine-id-cfgmap" \ + --set cortxcontrol.localpathpvc.name="cortx-control-fs-local-pvc" \ + --set cortxcontrol.localpathpvc.mountpath="$local_storage" \ + --set cortxcontrol.secretinfo="secret-info.txt" \ + --set cortxgluster.pv.name="gluster-default-name" \ --set cortxgluster.pv.mountpath=$shared_storage \ - --set cortxgluster.pvc.name=$gluster_pvc_name \ - --set cortxdata.cfgmap.name="cortx-cfgmap" \ - --set cortxdata.cfgmap.volmountname="config001-$node_name" \ - --set cortxdata.cfgmap.mountpath="/etc/cortx/solution" \ - --set cortxdata.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ - --set cortxdata.sslcfgmap.volmountname="ssl-config001" \ - --set cortxdata.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ - --set cortxdata.machineid.name="cortx-data-machine-id-cfgmap-$node_name" \ - --set cortxdata.localpathpvc.name="cortx-data-fs-local-pvc-$node_name" \ - --set cortxdata.localpathpvc.mountpath="$local_storage" \ - --set cortxdata.motr.numclientinst=$(extractBlock 'solution.common.motr.num_client_inst') \ - --set cortxdata.motr.numiosinst=${#cvg_index_list[@]} \ - --set cortxdata.motr.startportnum=$(extractBlock 'solution.common.motr.start_port_num') \ - --set cortxdata.s3.numinst=$(extractBlock 'solution.common.s3.num_inst') \ - --set cortxdata.s3.startportnum=$(extractBlock 'solution.common.s3.start_port_num') \ - --set cortxdata.secretinfo="secret-info.txt" \ + --set cortxgluster.pvc.name="gluster-claim" \ --set namespace=$namespace -done -printf "\nWait for CORTX Data to be ready" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[1]}" - if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then - if [[ "${pod_status[2]}" == "Error" ]]; then - printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" - exit 1 + printf "\nWait for CORTX Control to be ready" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[1]}" + if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + if [[ "${pod_status[2]}" == "Error" ]]; then + printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" + exit 1 + fi + break fi + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-control-pod-')" + + if [[ $num_nodes -eq $count ]]; then break + else + printf "." fi - count=$((count+1)) - done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-pod-')" + sleep 1s + done + printf "\n\n" +} - if [[ $num_nodes -eq $count ]]; then - break - else - printf "." +function deployCortxData() +{ + node_selector_list=$1 + node_name_list=$2 + gluster_pv_name=$3 + gluster_pvc_name=$4 + shared_storage=$5 + local_storage=$6 + cvg_index_list=$7 + namespace=$8 + printf "########################################################\n" + printf "# Deploy CORTX Data \n" + printf "########################################################\n" + cortxdata_image=$(parseSolution 'solution.images.cortxdata') + cortxdata_image=$(echo $cortxdata_image | cut -f2 -d'>') + + num_nodes=0 + for i in "${!node_selector_list[@]}"; do + num_nodes=$((num_nodes+1)) + node_name=${node_name_list[i]} + node_selector=${node_selector_list[i]} + helm install "cortx-data-$node_name" cortx-cloud-helm-pkg/cortx-data \ + --set cortxdata.name="cortx-data-pod-$node_name" \ + --set cortxdata.image=$cortxdata_image \ + --set cortxdata.nodename=$node_name \ + --set cortxdata.mountblkinfo="mnt-blk-info-$node_name.txt" \ + --set cortxdata.service.clusterip.name="cortx-data-clusterip-svc-$node_name" \ + --set cortxdata.service.headless.name="cortx-data-headless-svc-$node_name" \ + --set cortxdata.service.loadbal.name="cortx-data-loadbal-svc-$node_name" \ + --set cortxgluster.pv.name=$gluster_pv_name \ + --set cortxgluster.pv.mountpath=$shared_storage \ + --set cortxgluster.pvc.name=$gluster_pvc_name \ + --set cortxdata.cfgmap.name="cortx-cfgmap" \ + --set cortxdata.cfgmap.volmountname="config001-$node_name" \ + --set cortxdata.cfgmap.mountpath="/etc/cortx/solution" \ + --set cortxdata.sslcfgmap.name="cortx-ssl-cert-cfgmap" \ + --set cortxdata.sslcfgmap.volmountname="ssl-config001" \ + --set cortxdata.sslcfgmap.mountpath="/etc/cortx/solution/ssl" \ + --set cortxdata.machineid.name="cortx-data-machine-id-cfgmap-$node_name" \ + --set cortxdata.localpathpvc.name="cortx-data-fs-local-pvc-$node_name" \ + --set cortxdata.localpathpvc.mountpath="$local_storage" \ + --set cortxdata.motr.numclientinst=$(extractBlock 'solution.common.motr.num_client_inst') \ + --set cortxdata.motr.numiosinst=${#cvg_index_list[@]} \ + --set cortxdata.motr.startportnum=$(extractBlock 'solution.common.motr.start_port_num') \ + --set cortxdata.s3.numinst=$(extractBlock 'solution.common.s3.num_inst') \ + --set cortxdata.s3.startportnum=$(extractBlock 'solution.common.s3.start_port_num') \ + --set cortxdata.secretinfo="secret-info.txt" \ + --set namespace=$namespace + done + + printf "\nWait for CORTX Data to be ready" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[1]}" + if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + if [[ "${pod_status[2]}" == "Error" ]]; then + printf "\n'${pod_status[0]}' pod deployment did not complete. Exit early.\n" + exit 1 + fi + break + fi + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-pod-')" + + if [[ $num_nodes -eq $count ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n\n" +} + +function deployCortxServices() +{ + namespace=$1 + printf "########################################################\n" + printf "# Deploy Services \n" + printf "########################################################\n" + kubectl apply -f services/cortx-io-svc.yaml --namespace=$namespace + + cortx_io_svc_ingress=$(parseSolution 'solution.common.cortx_io_svc_ingress') + cortx_io_svc_ingress=$(echo $cortx_io_svc_ingress | cut -f2 -d'>') + if [ "$cortx_io_svc_ingress" == "true" ] + then + kubectl apply -f services/cortx-io-svc-ingress.yaml --namespace=$namespace fi - sleep 1s -done -printf "\n\n" +} -printf "########################################################\n" -printf "# Deploy Services \n" -printf "########################################################\n" -kubectl apply -f services/cortx-io-svc.yaml --namespace=$namespace +function deleteCortxProvisioners() +{ + namespace=$1 + printf "########################################################\n" + printf "# Delete CORTX Data provisioner \n" + printf "########################################################\n" + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + kubectl delete pod "${pod_status[0]}" --namespace=$namespace + count=$((count+1)) + done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-provisioner-pod-')" -cortx_io_svc_ingress=$(parseSolution 'solution.common.cortx_io_svc_ingress') -cortx_io_svc_ingress=$(echo $cortx_io_svc_ingress | cut -f2 -d'>') -if [ "$cortx_io_svc_ingress" == "true" ] -then - kubectl apply -f services/cortx-io-svc-ingress.yaml --namespace=$namespace + printf "########################################################\n" + printf "# Delete CORTX Control provisioner \n" + printf "########################################################\n" + kubectl delete pod cortx-control-provisioner-pod --namespace=$namespace +} + +function cleanup() +{ + ################################################################# + # Delete files that contain disk partitions on the worker nodes + # and the node info + ################################################################# + find $(pwd)/cortx-cloud-3rd-party-pkg/openldap -name "node-list-info*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "mnt-blk-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "mnt-blk-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-control-provisioner -name "secret-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-control -name "secret-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "secret-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "secret-*" -delete + + rm -rf "$cfgmap_path/auto-gen-secret" +} + +########################################################## +# Deploy CORTX 3rd party +########################################################## +# Extract storage provisioner path from the "solution.yaml" file +filter='solution.common.storage_provisioner_path' +parse_storage_prov_output=$(parseSolution $filter) +# Get the storage provisioner var from the tuple +storage_prov_path=$(echo $parse_storage_prov_output | cut -f2 -d'>') + +# Get number of consul replicas and make sure it doesn't exceed the limit +num_consul_replicas=$num_worker_nodes +if [[ "$num_worker_nodes" -gt "$max_consul_inst" ]]; then + num_consul_replicas=$max_consul_inst fi -printf "########################################################\n" -printf "# Delete CORTX Data provisioner \n" -printf "########################################################\n" -while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - kubectl delete pod "${pod_status[0]}" --namespace=$namespace - count=$((count+1)) -done <<< "$(kubectl get pods --namespace=$namespace | grep 'cortx-data-provisioner-pod-')" +# Get number of kafka replicas and make sure it doesn't exceed the limit +num_kafka_replicas=$num_worker_nodes +if [[ "$num_worker_nodes" -gt "$max_kafka_inst" ]]; then + num_kafka_replicas=$max_kafka_inst +fi -printf "########################################################\n" -printf "# Delete CORTX Control provisioner \n" -printf "########################################################\n" -kubectl delete pod cortx-control-provisioner-pod --namespace=$namespace +deployRancherProvisioner $storage_class $storage_prov_path +deployConsul $storage_class $num_consul_replicas +deployOpenLDAP $num_openldap_replicas +deployZookeeper $storage_class $num_kafka_replicas +deployKafka $num_worker_nodes $storage_class $num_kafka_replicas -################################################################# -# Delete files that contain disk partitions on the worker nodes -# and the node info -################################################################# -find $(pwd)/cortx-cloud-3rd-party-pkg/openldap -name "node-list-info*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "mnt-blk-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "mnt-blk-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-control-provisioner -name "secret-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-control -name "secret-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "secret-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "secret-*" -delete +########################################################## +# Deploy CORTX cloud +########################################################## +# Get the storage paths to use +local_storage=$(parseSolution 'solution.common.storage.local') +local_storage=$(echo $local_storage | cut -f2 -d'>') +shared_storage=$(parseSolution 'solution.common.storage.shared') +shared_storage=$(echo $shared_storage | cut -f2 -d'>') +log_storage=$(parseSolution 'solution.common.storage.log') +log_storage=$(echo $log_storage | cut -f2 -d'>') + +# GlusterFS +gluster_vol="myvol" +gluster_folder="/etc/gluster" +gluster_etc_path="$storage_prov_path/$gluster_folder" +gluster_pv_name="gluster-default-volume" +gluster_pvc_name="gluster-claim" + +# Default path to CORTX configmap +cfgmap_path="./cortx-cloud-helm-pkg/cortx-configmap" + +cvg_output=$(parseSolution 'solution.storage.cvg*.name') +IFS=';' read -r -a cvg_var_val_array <<< "$cvg_output" +# Build CVG index list (ex: [cvg1, cvg2, cvg3]) +cvg_index_list=[] +count=0 +for cvg_var_val_element in "${cvg_var_val_array[@]}"; do + cvg_name=$(echo $cvg_var_val_element | cut -f2 -d'>') + cvg_filter=$(echo $cvg_var_val_element | cut -f1 -d'>') + cvg_index=$(echo $cvg_filter | cut -f3 -d'.') + cvg_index_list[$count]=$cvg_index + count=$((count+1)) +done -rm -rf "$cfgmap_path/auto-gen-secret" +deployCortxLocalBlockStorage $node_selector_list $node_name_list $namespace +deployCortxGlusterFS $node_name_list $node_selector_list $gluster_vol $gluster_pv_name $gluster_pvc_name $gluster_etc_path $storage_prov_path $namespace +deleteStaleAutoGenFolders $node_name_list +deployCortxConfigMap $cfgmap_path $node_name_list $cvg_index_list $namespace +deployCortxSecrets $cfgmap_path $solution_yaml $namespace +deployCortxControlProvisioner $gluster_pv_name $gluster_pvc_name $shared_storage $local_storage $namespace +deployCortxDataProvisioner $node_selector_list $node_name_list $gluster_pv_name $gluster_pvc_name $shared_storage $local_storage $namespace +deployCortxControl $local_storage $shared_storage $namespace +deployCortxData $node_selector_list $node_name_list $gluster_pv_name $gluster_pvc_name $shared_storage $local_storage $cvg_index_list $namespace +deployCortxServices $namespace +deleteCortxProvisioners $namespace +cleanup \ No newline at end of file From c7a13119dab5e9534fc6301d917b4d7132dd5df7 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 27 Oct 2021 12:56:16 -0600 Subject: [PATCH 03/16] UDX-6040_helm_and_script_updates - split large content in pre-req and destroy scripts into functions - remove passing params into functions in deploy script since these are global variables --- k8_cortx_cloud/deploy-cortx-cloud.sh | 86 +-- k8_cortx_cloud/destroy-cortx-cloud.sh | 562 +++++++++++--------- k8_cortx_cloud/prereq-deploy-cortx-cloud.sh | 329 ++++++++---- 3 files changed, 549 insertions(+), 428 deletions(-) diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 0e7dd621..7cf28baf 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -177,8 +177,6 @@ fi ########################################################## function deployRancherProvisioner() { - storage_class=$1 - storage_prov_path=$2 # Add the HashiCorp Helm Repository: helm repo add hashicorp https://helm.releases.hashicorp.com if [[ $storage_class == "local-path" ]] @@ -199,8 +197,6 @@ function deployRancherProvisioner() function deployConsul() { - storage_class=$1 - num_consul_replicas=$2 printf "######################################################\n" printf "# Deploy Consul \n" printf "######################################################\n" @@ -216,7 +212,6 @@ function deployConsul() function deployOpenLDAP() { - num_openldap_replicas=$1 printf "######################################################\n" printf "# Deploy openLDAP \n" printf "######################################################\n" @@ -265,8 +260,6 @@ function deployOpenLDAP() function deployZookeeper() { - storage_class=$1 - num_kafka_replicas=$2 printf "######################################################\n" printf "# Deploy Zookeeper \n" printf "######################################################\n" @@ -282,9 +275,6 @@ function deployZookeeper() function deployKafka() { - num_worker_nodes=$1 - storage_class=$2 - num_kafka_replicas=$3 printf "######################################################\n" printf "# Deploy Kafka \n" printf "######################################################\n" @@ -328,9 +318,6 @@ function deployKafka() ########################################################## function deployCortxLocalBlockStorage() { - node_selector_list=$1 - node_name_list=$2 - namespace=$3 printf "######################################################\n" printf "# Deploy CORTX Local Block Storage \n" printf "######################################################\n" @@ -374,14 +361,6 @@ function deployCortxLocalBlockStorage() function deployCortxGlusterFS() { - node_name_list=$1 - node_selector_list=$2 - gluster_vol=$3 - gluster_pv_name=$4 - gluster_pvc_name=$5 - gluster_etc_path=$6 - storage_prov_path=$7 - namespace=$8 printf "########################################################\n" printf "# Deploy CORTX GlusterFS \n" printf "########################################################\n" @@ -499,7 +478,6 @@ function deployCortxGlusterFS() function deleteStaleAutoGenFolders() { - node_name_list=$1 # Delete all stale auto gen folders rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-cfgmap rm -rf $(pwd)/cortx-cloud-helm-pkg/cortx-configmap/auto-gen-control @@ -513,10 +491,6 @@ function deleteStaleAutoGenFolders() function deployCortxConfigMap() { - cfgmap_path=$1 - node_name_list=$2 - cvg_index_list=$3 - namespace=$4 printf "########################################################\n" printf "# Deploy CORTX Configmap \n" printf "########################################################\n" @@ -719,9 +693,6 @@ function deployCortxConfigMap() function deployCortxSecrets() { - cfgmap_path=$1 - solution_yaml=$2 - namespace=$3 printf "########################################################\n" printf "# Deploy CORTX Secrets \n" printf "########################################################\n" @@ -776,11 +747,6 @@ function deployCortxSecrets() function deployCortxControlProvisioner() { - gluster_pv_name=$1 - gluster_pvc_name=$2 - shared_storage=$3 - local_storage=$4 - namespace=$5 printf "########################################################\n" printf "# Deploy CORTX Control Provisioner \n" printf "########################################################\n" @@ -841,13 +807,6 @@ function deployCortxControlProvisioner() function deployCortxDataProvisioner() { - node_selector_list=$1 - node_name_list=$2 - gluster_pv_name=$3 - gluster_pvc_name=$4 - shared_storage=$5 - local_storage=$6 - namespace=$7 printf "########################################################\n" printf "# Deploy CORTX Data Provisioner \n" printf "########################################################\n" @@ -920,9 +879,6 @@ function deployCortxDataProvisioner() function deployCortxControl() { - local_storage=$1 - shared_storage=$2 - namespace=$3 printf "########################################################\n" printf "# Deploy CORTX Control \n" printf "########################################################\n" @@ -980,14 +936,6 @@ function deployCortxControl() function deployCortxData() { - node_selector_list=$1 - node_name_list=$2 - gluster_pv_name=$3 - gluster_pvc_name=$4 - shared_storage=$5 - local_storage=$6 - cvg_index_list=$7 - namespace=$8 printf "########################################################\n" printf "# Deploy CORTX Data \n" printf "########################################################\n" @@ -1056,7 +1004,6 @@ function deployCortxData() function deployCortxServices() { - namespace=$1 printf "########################################################\n" printf "# Deploy Services \n" printf "########################################################\n" @@ -1072,7 +1019,6 @@ function deployCortxServices() function deleteCortxProvisioners() { - namespace=$1 printf "########################################################\n" printf "# Delete CORTX Data provisioner \n" printf "########################################################\n" @@ -1126,11 +1072,11 @@ if [[ "$num_worker_nodes" -gt "$max_kafka_inst" ]]; then num_kafka_replicas=$max_kafka_inst fi -deployRancherProvisioner $storage_class $storage_prov_path -deployConsul $storage_class $num_consul_replicas -deployOpenLDAP $num_openldap_replicas -deployZookeeper $storage_class $num_kafka_replicas -deployKafka $num_worker_nodes $storage_class $num_kafka_replicas +deployRancherProvisioner +deployConsul +deployOpenLDAP +deployZookeeper +deployKafka ########################################################## # Deploy CORTX cloud @@ -1166,15 +1112,15 @@ for cvg_var_val_element in "${cvg_var_val_array[@]}"; do count=$((count+1)) done -deployCortxLocalBlockStorage $node_selector_list $node_name_list $namespace -deployCortxGlusterFS $node_name_list $node_selector_list $gluster_vol $gluster_pv_name $gluster_pvc_name $gluster_etc_path $storage_prov_path $namespace -deleteStaleAutoGenFolders $node_name_list -deployCortxConfigMap $cfgmap_path $node_name_list $cvg_index_list $namespace -deployCortxSecrets $cfgmap_path $solution_yaml $namespace -deployCortxControlProvisioner $gluster_pv_name $gluster_pvc_name $shared_storage $local_storage $namespace -deployCortxDataProvisioner $node_selector_list $node_name_list $gluster_pv_name $gluster_pvc_name $shared_storage $local_storage $namespace -deployCortxControl $local_storage $shared_storage $namespace -deployCortxData $node_selector_list $node_name_list $gluster_pv_name $gluster_pvc_name $shared_storage $local_storage $cvg_index_list $namespace -deployCortxServices $namespace -deleteCortxProvisioners $namespace +deployCortxLocalBlockStorage +deployCortxGlusterFS +deleteStaleAutoGenFolders +deployCortxConfigMap +deployCortxSecrets +deployCortxControlProvisioner +deployCortxDataProvisioner +deployCortxControl +deployCortxData +deployCortxServices +deleteCortxProvisioners cleanup \ No newline at end of file diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index 71018584..012da60c 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -76,226 +76,250 @@ do done ############################################################# -# Destroy CORTX Cloud +# Destroy CORTX Cloud functions ############################################################# +function deleteCortxData() +{ + printf "########################################################\n" + printf "# Delete CORTX Data \n" + printf "########################################################\n" + for i in "${!node_selector_list[@]}"; do + helm uninstall "cortx-data-${node_name_list[$i]}" + done +} -printf "########################################################\n" -printf "# Delete CORTX Data \n" -printf "########################################################\n" -for i in "${!node_selector_list[@]}"; do - helm uninstall "cortx-data-${node_name_list[$i]}" -done - -printf "########################################################\n" -printf "# Delete Services \n" -printf "########################################################\n" -kubectl delete service cortx-io-svc --namespace=$namespace - -cortx_io_svc_ingress=$(parseSolution 'solution.common.cortx_io_svc_ingress') -cortx_io_svc_ingress=$(echo $cortx_io_svc_ingress | cut -f2 -d'>') -if [ "$cortx_io_svc_ingress" == "true" ] -then - kubectl delete ingress cortx-io-svc-ingress --namespace=$namespace -fi +function deleteCortxServices() +{ + printf "########################################################\n" + printf "# Delete CORTX Services \n" + printf "########################################################\n" + kubectl delete service cortx-io-svc --namespace=$namespace + + cortx_io_svc_ingress=$(parseSolution 'solution.common.cortx_io_svc_ingress') + cortx_io_svc_ingress=$(echo $cortx_io_svc_ingress | cut -f2 -d'>') + if [ "$cortx_io_svc_ingress" == "true" ] + then + kubectl delete ingress cortx-io-svc-ingress --namespace=$namespace + fi +} -printf "########################################################\n" -printf "# Delete CORTX Control \n" -printf "########################################################\n" -helm uninstall "cortx-control" +function deleteCortxControl() +{ + printf "########################################################\n" + printf "# Delete CORTX Control \n" + printf "########################################################\n" + helm uninstall "cortx-control" +} -printf "########################################################\n" -printf "# Delete CORTX Data provisioner \n" -printf "########################################################\n" -for i in "${!node_selector_list[@]}"; do - helm uninstall "cortx-data-provisioner-${node_name_list[$i]}" -done +function deleteCortxProvisioners() +{ + printf "########################################################\n" + printf "# Delete CORTX Data provisioner \n" + printf "########################################################\n" + for i in "${!node_selector_list[@]}"; do + helm uninstall "cortx-data-provisioner-${node_name_list[$i]}" + done -printf "########################################################\n" -printf "# Delete CORTX Control provisioner \n" -printf "########################################################\n" -helm uninstall "cortx-control-provisioner" + printf "########################################################\n" + printf "# Delete CORTX Control provisioner \n" + printf "########################################################\n" + helm uninstall "cortx-control-provisioner" +} -printf "########################################################\n" -printf "# Delete CORTX GlusterFS \n" -printf "########################################################\n" -gluster_vol="myvol" +function deleteGlusterfs() +{ + printf "########################################################\n" + printf "# Delete CORTX GlusterFS \n" + printf "########################################################\n" + gluster_vol="myvol" + + # Build Gluster endpoint array + gluster_ep_array=[] + count=0 + while IFS= read -r line; do + if [[ $line == *"gluster-"* ]] + then + IFS=" " read -r -a my_array <<< "$line" + gluster_ep_array[count]=$line + count=$((count+1)) + fi + done <<< "$(kubectl get pods -A -o wide | grep 'gluster-')" -# Build Gluster endpoint array -gluster_ep_array=[] -count=0 -while IFS= read -r line; do - if [[ $line == *"gluster-"* ]] - then - IFS=" " read -r -a my_array <<< "$line" - gluster_ep_array[count]=$line + # Loop through all gluster endpoint array and find endoint IP address + # and gluster node name + count=0 + first_gluster_node_name='' + for gluster_ep in "${gluster_ep_array[@]}" + do + IFS=" " read -r -a my_array <<< "$gluster_ep" + gluster_ep_ip=${my_array[6]} + gluster_node_name=${my_array[1]} + printf "=================================================================================\n" + printf "Stop and delete GlusterFS volume: $gluster_node_name \n" + printf "=================================================================================\n" + kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ + 'rm -rf /etc/gluster/* /etc/gluster/.glusterfs/' + kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ + 'mkdir -p /etc/gluster/var/log/cortx' + if [[ "$count" == 0 ]]; then + first_gluster_node_name=$gluster_node_name + echo y | kubectl exec --namespace=$namespace -i $gluster_node_name -- gluster volume stop $gluster_vol + echo y | kubectl exec --namespace=$namespace -i $gluster_node_name -- gluster volume delete $gluster_vol + else + echo y | kubectl exec --namespace=$namespace -i $first_gluster_node_name -- gluster peer detach $gluster_ep_ip + fi count=$((count+1)) - fi -done <<< "$(kubectl get pods -A -o wide | grep 'gluster-')" + done -# Loop through all gluster endpoint array and find endoint IP address -# and gluster node name -count=0 -first_gluster_node_name='' -for gluster_ep in "${gluster_ep_array[@]}" -do - IFS=" " read -r -a my_array <<< "$gluster_ep" - gluster_ep_ip=${my_array[6]} - gluster_node_name=${my_array[1]} - printf "=================================================================================\n" - printf "Stop and delete GlusterFS volume: $gluster_node_name \n" - printf "=================================================================================\n" - kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ - 'rm -rf /etc/gluster/* /etc/gluster/.glusterfs/' - kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ - 'mkdir -p /etc/gluster/var/log/cortx' - if [[ "$count" == 0 ]]; then - first_gluster_node_name=$gluster_node_name - echo y | kubectl exec --namespace=$namespace -i $gluster_node_name -- gluster volume stop $gluster_vol - echo y | kubectl exec --namespace=$namespace -i $gluster_node_name -- gluster volume delete $gluster_vol - else - echo y | kubectl exec --namespace=$namespace -i $first_gluster_node_name -- gluster peer detach $gluster_ep_ip - fi - count=$((count+1)) -done + while IFS= read -r line; do + IFS=" " read -r -a my_array <<< "$line" + helm uninstall ${my_array[0]} + done <<< "$(helm ls | grep 'cortx-gluster')" +} -while IFS= read -r line; do - IFS=" " read -r -a my_array <<< "$line" - helm uninstall ${my_array[0]} -done <<< "$(helm ls | grep 'cortx-gluster')" - -printf "######################################################\n" -printf "# Delete CORTX Local Block Storage \n" -printf "######################################################\n" -for i in "${!node_selector_list[@]}"; do - node_name=${node_name_list[i]} - node_selector=${node_selector_list[i]} - file_path="cortx-cloud-helm-pkg/cortx-data-provisioner/mnt-blk-info-$node_name.txt" - count=001 - while IFS=' ' read -r mount_path || [[ -n "$mount_path" ]]; do - count_str=$(printf "%03d" $count) - count=$((count+1)) - helm_name1="cortx-data-blk-data$count_str-$node_name" - helm uninstall $helm_name1 - done < "$file_path" -done +function deleteCortxLocalBlockStorage() +{ + printf "######################################################\n" + printf "# Delete CORTX Local Block Storage \n" + printf "######################################################\n" + for i in "${!node_selector_list[@]}"; do + node_name=${node_name_list[i]} + node_selector=${node_selector_list[i]} + file_path="cortx-cloud-helm-pkg/cortx-data-provisioner/mnt-blk-info-$node_name.txt" + count=001 + while IFS=' ' read -r mount_path || [[ -n "$mount_path" ]]; do + count_str=$(printf "%03d" $count) + count=$((count+1)) + helm_name1="cortx-data-blk-data$count_str-$node_name" + helm uninstall $helm_name1 + done < "$file_path" + done +} -printf "######################################################\n" -printf "# Delete Persistent Volume \n" -printf "######################################################\n" -while IFS= read -r line; do - if [[ $line != *"master"* && $line != *"AGE"* ]] - then - IFS=" " read -r -a pvc_line <<< "$line" - if [[ "${pvc_line[5]}" == *"cortx-"* && "${pvc_line[5]}" != *"openldap"* ]]; then - printf "Removing ${pvc_line[0]}\n" - if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then - kubectl patch pv ${pvc_line[0]} -p '{"metadata":{"finalizers":null}}' +function deleteCortxPVs() +{ + printf "######################################################\n" + printf "# Delete CORTX Persistent Volumes \n" + printf "######################################################\n" + while IFS= read -r line; do + if [[ $line != *"master"* && $line != *"AGE"* ]] + then + IFS=" " read -r -a pvc_line <<< "$line" + if [[ "${pvc_line[5]}" == *"cortx-"* && "${pvc_line[5]}" != *"openldap"* ]]; then + printf "Removing ${pvc_line[0]}\n" + if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then + kubectl patch pv ${pvc_line[0]} -p '{"metadata":{"finalizers":null}}' + fi + kubectl delete pv ${pvc_line[0]} fi - kubectl delete pv ${pvc_line[0]} fi - fi -done <<< "$(kubectl get pv -A)" - -printf "########################################################\n" -printf "# Delete CORTX Configmap #\n" -printf "########################################################\n" -cfgmap_path="./cortx-cloud-helm-pkg/cortx-configmap" -# Delete data machine id config maps -for i in "${!node_name_list[@]}"; do - kubectl delete configmap "cortx-data-machine-id-cfgmap-${node_name_list[i]}" --namespace=$namespace - rm -rf "$cfgmap_path/auto-gen-${node_name_list[i]}" - -done -# Delete control machine id config map -kubectl delete configmap "cortx-control-machine-id-cfgmap" --namespace=$namespace -rm -rf "$cfgmap_path/auto-gen-control" -# Delete CORTX config maps -rm -rf "$cfgmap_path/auto-gen-cfgmap" -kubectl delete configmap "cortx-cfgmap" --namespace=$namespace -rm -rf "$cfgmap_path/auto-gen-cfgmap" + done <<< "$(kubectl get pv -A)" +} -rm -rf "$cfgmap_path/node-info" -rm -rf "$cfgmap_path/storage-info" +function deleteCortxConfigmap() +{ + printf "########################################################\n" + printf "# Delete CORTX Configmap #\n" + printf "########################################################\n" + cfgmap_path="./cortx-cloud-helm-pkg/cortx-configmap" + # Delete data machine id config maps + for i in "${!node_name_list[@]}"; do + kubectl delete configmap "cortx-data-machine-id-cfgmap-${node_name_list[i]}" --namespace=$namespace + rm -rf "$cfgmap_path/auto-gen-${node_name_list[i]}" -# Delete SSL cert config map -ssl_cert_path="$cfgmap_path/ssl-cert" -kubectl delete configmap "cortx-ssl-cert-cfgmap" --namespace=$namespace + done + # Delete control machine id config map + kubectl delete configmap "cortx-control-machine-id-cfgmap" --namespace=$namespace + rm -rf "$cfgmap_path/auto-gen-control" + # Delete CORTX config maps + rm -rf "$cfgmap_path/auto-gen-cfgmap" + kubectl delete configmap "cortx-cfgmap" --namespace=$namespace + rm -rf "$cfgmap_path/auto-gen-cfgmap" + + rm -rf "$cfgmap_path/node-info" + rm -rf "$cfgmap_path/storage-info" + + # Delete SSL cert config map + ssl_cert_path="$cfgmap_path/ssl-cert" + kubectl delete configmap "cortx-ssl-cert-cfgmap" --namespace=$namespace +} ############################################################# -# Destroy CORTX 3rd party +# Destroy CORTX 3rd party functions ############################################################# +function deleteKafkaZookeper() +{ + printf "########################################################\n" + printf "# Delete Kafka #\n" + printf "########################################################\n" + helm uninstall kafka + + printf "########################################################\n" + printf "# Delete Zookeeper #\n" + printf "########################################################\n" + helm uninstall zookeeper +} -printf "########################################################\n" -printf "# Delete Kafka #\n" -printf "########################################################\n" -helm uninstall kafka - -printf "########################################################\n" -printf "# Delete Zookeeper #\n" -printf "########################################################\n" -helm uninstall zookeeper +function deleteOpenLdap() +{ + printf "########################################################\n" + printf "# Delete openLDAP #\n" + printf "########################################################\n" + openldap_array=[] + count=0 + while IFS= read -r line; do + IFS=" " read -r -a my_array <<< "$line" + openldap_array[count]="${my_array[1]}" + count=$((count+1)) + done <<< "$(kubectl get pods -A | grep 'openldap-')" -printf "########################################################\n" -printf "# Delete openLDAP #\n" -printf "########################################################\n" -openldap_array=[] -count=0 -while IFS= read -r line; do - IFS=" " read -r -a my_array <<< "$line" - openldap_array[count]="${my_array[1]}" - count=$((count+1)) -done <<< "$(kubectl get pods -A | grep 'openldap-')" + for openldap_pod_name in "${openldap_array[@]}" + do + kubectl exec -ti $openldap_pod_name --namespace="default" -- bash -c \ + 'rm -rf /etc/3rd-party/* /var/data/3rd-party/* /var/log/3rd-party/*' + done -for openldap_pod_name in "${openldap_array[@]}" -do - kubectl exec -ti $openldap_pod_name --namespace="default" -- bash -c \ - 'rm -rf /etc/3rd-party/* /var/data/3rd-party/* /var/log/3rd-party/*' -done + helm uninstall "openldap" +} -helm uninstall "openldap" +function deleteSecrets() +{ + printf "########################################################\n" + printf "# Delete Secrets #\n" + printf "########################################################\n" + output=$(./parse_scripts/parse_yaml.sh $solution_yaml "solution.secrets*.name") + IFS=';' read -r -a parsed_secret_name_array <<< "$output" + for secret_name in "${parsed_secret_name_array[@]}" + do + secret_name=$(echo $secret_name | cut -f2 -d'>') + kubectl delete secret $secret_name --namespace=$namespace + done -printf "########################################################\n" -printf "# Delete Secrets #\n" -printf "########################################################\n" -output=$(./parse_scripts/parse_yaml.sh $solution_yaml "solution.secrets*.name") -IFS=';' read -r -a parsed_secret_name_array <<< "$output" -for secret_name in "${parsed_secret_name_array[@]}" -do - secret_name=$(echo $secret_name | cut -f2 -d'>') - kubectl delete secret $secret_name --namespace=$namespace -done + find $(pwd)/cortx-cloud-helm-pkg/cortx-control-provisioner -name "secret-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-control -name "secret-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "secret-*" -delete + find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "secret-*" -delete +} -find $(pwd)/cortx-cloud-helm-pkg/cortx-control-provisioner -name "secret-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-control -name "secret-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner -name "secret-*" -delete -find $(pwd)/cortx-cloud-helm-pkg/cortx-data -name "secret-*" -delete - -printf "########################################################\n" -printf "# Delete Consul #\n" -printf "########################################################\n" -helm delete consul - -rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" -rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" -kubectl delete -f $rancher_prov_file -rm -rf $rancher_prov_path - -printf "########################################################\n" -printf "# Delete Persistent Volume Claims #\n" -printf "########################################################\n" -volume_claims=$(kubectl get pvc --namespace=default | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter|$openldap_pvc|cortx|3rd-party" | cut -f1 -d " ") -echo $volume_claims -for volume_claim in $volume_claims -do - printf "Removing $volume_claim\n" - if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then - kubectl patch pvc $volume_claim -p '{"metadata":{"finalizers":null}}' - fi - kubectl delete pvc $volume_claim -done +function deleteConsul() +{ + printf "########################################################\n" + printf "# Delete Consul #\n" + printf "########################################################\n" + helm delete consul + + rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" + rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" + kubectl delete -f $rancher_prov_file + rm -rf $rancher_prov_path +} -if [[ $namespace != 'default' ]]; then - volume_claims=$(kubectl get pvc --namespace=$namespace | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter|$openldap_pvc|cortx|3rd-party" | cut -f1 -d " ") +function delete3rdPartyPVCs() +{ + printf "########################################################\n" + printf "# Delete Persistent Volume Claims #\n" + printf "########################################################\n" + volume_claims=$(kubectl get pvc --namespace=default | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter|$openldap_pvc|cortx|3rd-party" | cut -f1 -d " ") echo $volume_claims for volume_claim in $volume_claims do @@ -305,66 +329,120 @@ if [[ $namespace != 'default' ]]; then fi kubectl delete pvc $volume_claim done -fi -printf "########################################################\n" -printf "# Delete Persistent Volumes #\n" -printf "########################################################\n" -persistent_volumes=$(kubectl get pv --namespace=default | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter" | cut -f1 -d " ") -echo $persistent_volumes -for persistent_volume in $persistent_volumes -do - printf "Removing $persistent_volume\n" - if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then - kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' + if [[ $namespace != 'default' ]]; then + volume_claims=$(kubectl get pvc --namespace=$namespace | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter|$openldap_pvc|cortx|3rd-party" | cut -f1 -d " ") + echo $volume_claims + for volume_claim in $volume_claims + do + printf "Removing $volume_claim\n" + if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then + kubectl patch pvc $volume_claim -p '{"metadata":{"finalizers":null}}' + fi + kubectl delete pvc $volume_claim + done fi - kubectl delete pv $persistent_volume -done +} -if [[ $namespace != 'default' ]]; then - persistent_volumes=$(kubectl get pv --namespace=$namespace | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter" | cut -f1 -d " ") +function delete3rdPartyPVs() +{ + printf "########################################################\n" + printf "# Delete Persistent Volumes #\n" + printf "########################################################\n" + persistent_volumes=$(kubectl get pv --namespace=default | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter" | cut -f1 -d " ") echo $persistent_volumes for persistent_volume in $persistent_volumes do - printf "Removing $persistent_volume\n" + printf "Removing $persistent_volume\n" if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' fi kubectl delete pv $persistent_volume done -fi -print_header=true -helm_ls_header=true -while IFS= read -r line; do - IFS=" " read -r -a my_array <<< "$line" - if [[ "$helm_ls_header" = true ]]; then - helm_ls_header=false - continue + if [[ $namespace != 'default' ]]; then + persistent_volumes=$(kubectl get pv --namespace=$namespace | grep -E "$pvc_consul_filter|$pvc_kafka_filter|$pvc_zookeeper_filter" | cut -f1 -d " ") + echo $persistent_volumes + for persistent_volume in $persistent_volumes + do + printf "Removing $persistent_volume\n" + if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then + kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' + fi + kubectl delete pv $persistent_volume + done fi - if [[ "$print_header" = true ]]; then - printf "Helm chart cleanup:\n" - print_header=false +} + +function helmChartCleanup() +{ + print_header=true + helm_ls_header=true + while IFS= read -r line; do + IFS=" " read -r -a my_array <<< "$line" + if [[ "$helm_ls_header" = true ]]; then + helm_ls_header=false + continue + fi + if [[ "$print_header" = true ]]; then + printf "Helm chart cleanup:\n" + print_header=false + fi + helm uninstall ${my_array[0]} + done <<< "$(helm ls | grep 'consul\|cortx\|kafka\|openldap\|zookeeper')" +} + +function deleteCortxNamespace() +{ + # Delete CORTX namespace + if [[ "$namespace" != "default" ]]; then + kubectl delete namespace $namespace fi - helm uninstall ${my_array[0]} -done <<< "$(helm ls | grep 'consul\|cortx\|kafka\|openldap\|zookeeper')" +} -# Delete CORTX namespace -if [[ "$namespace" != "default" ]]; then - kubectl delete namespace $namespace -fi +function cleanup() +{ + ################################################################# + # Delete files that contain disk partitions on the worker nodes # + ################################################################# + # Split parsed output into an array of vars and vals + IFS=';' read -r -a parsed_var_val_array <<< "$parsed_node_output" + # Loop the var val tuple array + for var_val_element in "${parsed_var_val_array[@]}" + do + node_name=$(echo $var_val_element | cut -f2 -d'>') + shorter_node_name=$(echo $node_name | cut -f1 -d'.') + file_name="mnt-blk-info-$shorter_node_name.txt" + rm $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner/$file_name + rm $(pwd)/cortx-cloud-helm-pkg/cortx-data/$file_name + done +} -################################################################# -# Delete files that contain disk partitions on the worker nodes # -################################################################# -# Split parsed output into an array of vars and vals -IFS=';' read -r -a parsed_var_val_array <<< "$parsed_node_output" -# Loop the var val tuple array -for var_val_element in "${parsed_var_val_array[@]}" -do - node_name=$(echo $var_val_element | cut -f2 -d'>') - shorter_node_name=$(echo $node_name | cut -f1 -d'.') - file_name="mnt-blk-info-$shorter_node_name.txt" - rm $(pwd)/cortx-cloud-helm-pkg/cortx-data-provisioner/$file_name - rm $(pwd)/cortx-cloud-helm-pkg/cortx-data/$file_name -done +############################################################# +# Destroy CORTX Cloud +############################################################# +deleteCortxData +deleteCortxServices +deleteCortxControl +deleteCortxProvisioners +deleteGlusterfs +deleteCortxLocalBlockStorage +deleteCortxPVs +deleteCortxConfigmap + +############################################################# +# Destroy CORTX 3rd party +############################################################# +deleteKafkaZookeper +deleteOpenLdap +deleteSecrets +deleteConsul +delete3rdPartyPVCs +delete3rdPartyPVs + +############################################################# +# Clean up +############################################################# +helmChartCleanup +deleteCortxNamespace +cleanup \ No newline at end of file diff --git a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh index 12e2ba0e..39e03209 100755 --- a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh @@ -10,11 +10,6 @@ then exit 1 fi -function parseSolution() -{ - echo "$(./parse_scripts/parse_yaml.sh $solution_yaml $1)" -} - fs_mount_path="/mnt/fs-local-volume" if [[ "$disk" == "" ]] @@ -24,127 +19,229 @@ then exit 1 fi -printf "####################################################\n" -printf "# Install helm \n" -printf "####################################################\n" -curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 -chmod 700 get_helm.sh -./get_helm.sh - -printf "####################################################\n" -printf "# Pull required docker images \n" -printf "####################################################\n" -# pull docker 3rd party images - -image=$(parseSolution 'solution.images.cortxcontrolprov') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi +function parseYaml +{ + s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') + sed -ne "s|^\($s\):|\1|" \ + -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | + awk -F$fs '{ + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i%s;",vn, $2, $3); + } + }' +} -image=$(parseSolution 'solution.images.cortxcontrol') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi +function parseSolution() +{ + # Check that all of the required parameters have been passed in + if [ "$solution_yaml" == "" ] + then + echo "Invalid input paramters" + echo " = $solution_yaml" + echo "[ OPTIONAL] = $2" + exit 1 + fi -image=$(parseSolution 'solution.images.cortxdataprov') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi + # Check if the file exists + if [ ! -f $solution_yaml ] + then + echo "ERROR: $solution_yaml does not exist" + exit 1 + fi -image=$(parseSolution 'solution.images.cortxdata') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi + # Store the parsed output in a single string + PARSED_OUTPUT=$(parseYaml $solution_yaml) + # Remove any additional indent '.' characters + PARSED_OUTPUT=$(echo ${PARSED_OUTPUT//../.}) -image=$(parseSolution 'solution.images.openldap') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi + # Star with empty output + OUTPUT="" -image=$(parseSolution 'solution.images.consul') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi + # Check if we need to do any filtering + if [ "$2" == "" ] + then + OUTPUT=$PARSED_OUTPUT + else + # Split parsed output into an array of vars and vals + IFS=';' read -r -a PARSED_VAR_VAL_ARRAY <<< "$PARSED_OUTPUT" + # Loop the var val tuple array + for VAR_VAL_ELEMENT in "${PARSED_VAR_VAL_ARRAY[@]}" + do + # Get the var and val from the tuple + VAR=$(echo $VAR_VAL_ELEMENT | cut -f1 -d'>') + # Check is the filter matches the var + if [[ $VAR == $2 ]] + then + # If the OUTPUT is empty set it otherwise append + if [ "$OUTPUT" == "" ] + then + OUTPUT=$VAR_VAL_ELEMENT + else + OUTPUT=$OUTPUT";"$VAR_VAL_ELEMENT + fi + fi + done + fi -image=$(parseSolution 'solution.images.kafka') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi + # Return the parsed output + echo $OUTPUT +} -image=$(parseSolution 'solution.images.zookeeper') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi +function installHelm() +{ + printf "####################################################\n" + printf "# Install helm \n" + printf "####################################################\n" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh +} -image=$(parseSolution 'solution.images.gluster') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi +function pullRequiredDockerImages() +{ + printf "####################################################\n" + printf "# Pull required docker images \n" + printf "####################################################\n" + # pull docker 3rd party images + + image=$(parseSolution $solution_yaml 'solution.images.cortxcontrolprov') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi -image=$(parseSolution 'solution.images.rancher') -image=$(echo $image | cut -f2 -d'>') -if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image -fi + image=$(parseSolution $solution_yaml 'solution.images.cortxcontrol') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi -# Pull the latest busybox image -docker pull busybox - -printf "####################################################\n" -printf "# Clean up \n" -printf "####################################################\n" -# cleanup -rm -rf /etc/3rd-party/openldap /var/data/3rd-party/* -rm -rf $fs_mount_path/local-path-provisioner/* -rm -rf $fs_mount_path/etc/gluster/var/log/cortx/* - -# Increase Resources -sysctl -w vm.max_map_count=30000000; - -printf "####################################################\n" -printf "# Prep for CORTX deployment \n" -printf "####################################################\n" - -if [[ $(findmnt -m $fs_mount_path) ]];then - echo "$fs_mount_path already mounted..." -else - mkdir -p $fs_mount_path - echo y | mkfs.ext4 $disk - mount -t ext4 $disk $fs_mount_path -fi + image=$(parseSolution $solution_yaml 'solution.images.cortxdataprov') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + image=$(parseSolution $solution_yaml 'solution.images.cortxdata') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi -# Prep for Rancher Local Path Provisioner deployment -echo "Create folder '$fs_mount_path/local-path-provisioner'" -mkdir -p $fs_mount_path/local-path-provisioner -count=0 -while true; do - if [[ -d $fs_mount_path/local-path-provisioner || $count -gt 5 ]]; then - break + image=$(parseSolution $solution_yaml 'solution.images.openldap') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + image=$(parseSolution $solution_yaml 'solution.images.consul') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + image=$(parseSolution $solution_yaml 'solution.images.kafka') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + image=$(parseSolution $solution_yaml 'solution.images.zookeeper') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + image=$(parseSolution $solution_yaml 'solution.images.gluster') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + image=$(parseSolution $solution_yaml 'solution.images.rancher') + image=$(echo $image | cut -f2 -d'>') + if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then + docker pull $image + fi + + # Pull the latest busybox image + docker pull busybox +} + +function cleanupFolders() +{ + printf "####################################################\n" + printf "# Clean up \n" + printf "####################################################\n" + # cleanup + rm -rf /etc/3rd-party/openldap /var/data/3rd-party/* + rm -rf $fs_mount_path/local-path-provisioner/* + rm -rf $fs_mount_path/etc/gluster/var/log/cortx/* +} + +function increaseResources() +{ + # Increase Resources + sysctl -w vm.max_map_count=30000000; +} + +function prepCortxDeployment() +{ + printf "####################################################\n" + printf "# Prep for CORTX deployment \n" + printf "####################################################\n" + + if [[ $(findmnt -m $fs_mount_path) ]];then + echo "$fs_mount_path already mounted..." else - echo "Create folder '$fs_mount_path/local-path-provisioner' failed. Retry..." - mkdir -p $fs_mount_path/local-path-provisioner + mkdir -p $fs_mount_path + echo y | mkfs.ext4 $disk + mount -t ext4 $disk $fs_mount_path fi - count=$((count+1)) - sleep 1s -done - -# Prep for GlusterFS deployment -yum install glusterfs-fuse -y -mkdir -p $fs_mount_path/etc/gluster -mkdir -p $fs_mount_path/etc/gluster/var/log/cortx -mkdir -p $fs_mount_path/var/log/glusterfs -mkdir -p $fs_mount_path/var/lib/glusterd - -# Prep for OpenLDAP deployment -mkdir -p /etc/3rd-party/openldap -mkdir -p /var/data/3rd-party -mkdir -p /var/log/3rd-party \ No newline at end of file + + # Prep for Rancher Local Path Provisioner deployment + echo "Create folder '$fs_mount_path/local-path-provisioner'" + mkdir -p $fs_mount_path/local-path-provisioner + count=0 + while true; do + if [[ -d $fs_mount_path/local-path-provisioner || $count -gt 5 ]]; then + break + else + echo "Create folder '$fs_mount_path/local-path-provisioner' failed. Retry..." + mkdir -p $fs_mount_path/local-path-provisioner + fi + count=$((count+1)) + sleep 1s + done +} + +function prepGlusterfsDeployment() +{ + # Prep for GlusterFS deployment + yum install glusterfs-fuse -y + mkdir -p $fs_mount_path/etc/gluster + mkdir -p $fs_mount_path/etc/gluster/var/log/cortx + mkdir -p $fs_mount_path/var/log/glusterfs + mkdir -p $fs_mount_path/var/lib/glusterd +} + +function prepOpenLdapDeployment() +{ + # Prep for OpenLDAP deployment + mkdir -p /etc/3rd-party/openldap + mkdir -p /var/data/3rd-party + mkdir -p /var/log/3rd-party +} + +installHelm +pullRequiredDockerImages +cleanupFolders +increaseResources +prepCortxDeployment +prepGlusterfsDeployment +prepOpenLdapDeployment \ No newline at end of file From 39246e725a14385d73b4e2a9dbe46b69469a108f Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:54:32 -0600 Subject: [PATCH 04/16] UDX-6040_helm_and_script_updates - update README.txt - change solution.common.storage to solution.common.container_path in solution.yaml and solution_dummy.yaml - remove installHelm and pullRequiredDockerImages from prereq-deploy-cortx-cloud.sh --- k8_cortx_cloud/README.txt | 37 ++++++--- k8_cortx_cloud/deploy-cortx-cloud.sh | 16 ++-- k8_cortx_cloud/prereq-deploy-cortx-cloud.sh | 89 ++------------------- k8_cortx_cloud/solution.yaml | 42 +++++++++- k8_cortx_cloud/solution_dummy.yaml | 2 +- 5 files changed, 81 insertions(+), 105 deletions(-) diff --git a/k8_cortx_cloud/README.txt b/k8_cortx_cloud/README.txt index 1f911789..952f4af0 100644 --- a/k8_cortx_cloud/README.txt +++ b/k8_cortx_cloud/README.txt @@ -1,3 +1,13 @@ + +############################################### +# Install helm # +############################################### +1. Install helm on the master node: + +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh + ############################################### # Local block storage requirements # ############################################### @@ -15,20 +25,26 @@ cluster ############################################### # Run prerequisite deployment script # ############################################### -1. Install the entire content of "k8_cortx_cloud" on both master node and all worker - nodes. The directory structure must be maintained. +1. Copy "prereq-deploy-cortx-cloud.sh" script, and the solution yaml file to all worker nodes: + +scp prereq-deploy-cortx-cloud.sh @: +scp @: + +Example: +scp prereq-deploy-cortx-cloud.sh root@192.168.1.1:/home/ +scp solution.yaml root@192.168.1.1:/home/ 2. Run prerequisite script on all worker nodes in the cluster, and untainted master node that allows scheduling. "" is a required input to run this script. This disk should NOT be any of the devices listed in "solution.storage.cvg*" in the "solution.yaml" file: -./prereq-deploy-cortx-cloud.sh [] +sudo ./prereq-deploy-cortx-cloud.sh [] Example: -./prereq-deploy-cortx-cloud.sh /dev/sdb +sudo ./prereq-deploy-cortx-cloud.sh /dev/sdb or -./prereq-deploy-cortx-cloud.sh /dev/sdb solution_dummy.yaml +sudo ./prereq-deploy-cortx-cloud.sh /dev/sdb solution_dummy.yaml NOTE: is an optional input to run "prereq-deploy-cortx-cloud.sh" script. Make sure to use @@ -39,13 +55,13 @@ the same solution file for pre-req, deploy and destroy scripts (in the below sec # Deploy and destroy CORTX cloud # ############################################### 1. Deploy CORTX cloud: -./deploy-cortx-cloud.sh [] +sudo ./deploy-cortx-cloud.sh [] 2. Destroy CORTX cloud: -./destroy-cortx-cloud.sh [] [--force|-f] +sudo ./destroy-cortx-cloud.sh [] [--force|-f] Example: -./destroy-cortx-cloud.sh solution.yaml --force +sudo ./destroy-cortx-cloud.sh solution.yaml --force NOTE: is an optional input to run deploy and destroy scripts. Make sure to use the same @@ -127,4 +143,7 @@ solution: kafka: bitnami/kafka:3.0.0-debian-10-r7 zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 gluster: docker.io/gluster/gluster-centos:latest - rancher: rancher/local-path-provisioner:v0.0.20 \ No newline at end of file + rancher: rancher/local-path-provisioner:v0.0.20 + +NOTE: These images can be pre-downloaded on all worker nodes and untainted master node that +allows scheduling to avoid deployment failure due to docker pull rate limits. \ No newline at end of file diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 7cf28baf..c10237e0 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -534,9 +534,9 @@ function deployCortxConfigMap() ./parse_scripts/subst.sh $new_gen_file "cortx.io.svc" "cortx-io-svc" ./parse_scripts/subst.sh $new_gen_file "cortx.num_s3_inst" $(extractBlock 'solution.common.s3.num_inst') ./parse_scripts/subst.sh $new_gen_file "cortx.num_motr_inst" $(extractBlock 'solution.common.motr.num_client_inst') - ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.local" $(extractBlock 'solution.common.storage.local') - ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.shared" $(extractBlock 'solution.common.storage.shared') - ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.log" $(extractBlock 'solution.common.storage.log') + ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.local" $local_storage + ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.shared" $shared_storage + ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.log" $log_storage # Generate node file with type storage_node in "node-info" folder new_gen_file="$node_info_folder/cluster-storage-node-${node_name_list[$i]}.yaml" cp "$cfgmap_path/templates/cluster-node-template.yaml" $new_gen_file @@ -903,9 +903,9 @@ function deployCortxControl() --set cortxcontrol.localpathpvc.name="cortx-control-fs-local-pvc" \ --set cortxcontrol.localpathpvc.mountpath="$local_storage" \ --set cortxcontrol.secretinfo="secret-info.txt" \ - --set cortxgluster.pv.name="gluster-default-name" \ + --set cortxgluster.pv.name=$gluster_pv_name \ --set cortxgluster.pv.mountpath=$shared_storage \ - --set cortxgluster.pvc.name="gluster-claim" \ + --set cortxgluster.pvc.name=$gluster_pvc_name \ --set namespace=$namespace printf "\nWait for CORTX Control to be ready" @@ -1082,11 +1082,11 @@ deployKafka # Deploy CORTX cloud ########################################################## # Get the storage paths to use -local_storage=$(parseSolution 'solution.common.storage.local') +local_storage=$(parseSolution 'solution.common.container_path.local') local_storage=$(echo $local_storage | cut -f2 -d'>') -shared_storage=$(parseSolution 'solution.common.storage.shared') +shared_storage=$(parseSolution 'solution.common.container_path.shared') shared_storage=$(echo $shared_storage | cut -f2 -d'>') -log_storage=$(parseSolution 'solution.common.storage.log') +log_storage=$(parseSolution 'solution.common.container_path.log') log_storage=$(echo $log_storage | cut -f2 -d'>') # GlusterFS diff --git a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh index 39e03209..c027c687 100755 --- a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh @@ -10,7 +10,11 @@ then exit 1 fi -fs_mount_path="/mnt/fs-local-volume" +# Extract storage provisioner path from the "solution.yaml" file +filter='solution.common.storage_provisioner_path' +parse_storage_prov_output=$(parseSolution $filter) +# Get the storage provisioner var from the tuple +fs_mount_path=$(echo $parse_storage_prov_output | cut -f2 -d'>') if [[ "$disk" == "" ]] then @@ -92,87 +96,6 @@ function parseSolution() echo $OUTPUT } -function installHelm() -{ - printf "####################################################\n" - printf "# Install helm \n" - printf "####################################################\n" - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh -} - -function pullRequiredDockerImages() -{ - printf "####################################################\n" - printf "# Pull required docker images \n" - printf "####################################################\n" - # pull docker 3rd party images - - image=$(parseSolution $solution_yaml 'solution.images.cortxcontrolprov') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.cortxcontrol') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.cortxdataprov') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.cortxdata') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.openldap') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.consul') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.kafka') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.zookeeper') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.gluster') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - image=$(parseSolution $solution_yaml 'solution.images.rancher') - image=$(echo $image | cut -f2 -d'>') - if [[ "$(docker images -q $image 2> /dev/null)" == "" ]]; then - docker pull $image - fi - - # Pull the latest busybox image - docker pull busybox -} - function cleanupFolders() { printf "####################################################\n" @@ -238,8 +161,6 @@ function prepOpenLdapDeployment() mkdir -p /var/log/3rd-party } -installHelm -pullRequiredDockerImages cleanupFolders increaseResources prepCortxDeployment diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index 27f67351..e13f8d4e 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -24,7 +24,7 @@ solution: common: cortx_io_svc_ingress: false storage_provisioner_path: /mnt/fs-local-volume - storage: + container_path: local: /etc/cortx shared: /share log: /share/var/log/cortx @@ -53,16 +53,52 @@ solution: d1: device: /dev/sdd size: 5Gi + d2: + device: /dev/sde + size: 5Gi + d3: + device: /dev/sdf + size: 5Gi + d4: + device: /dev/sdg + size: 5Gi + d5: + device: /dev/sdh + size: 5Gi + d6: + device: /dev/sdi + size: 5Gi + d7: + device: /dev/sdj + size: 5Gi cvg2: name: cvg-02 type: ios devices: metadata: - device: /dev/sde + device: /dev/sdk size: 5Gi data: d1: - device: /dev/sdf + device: /dev/sdl + size: 5Gi + d2: + device: /dev/sdm + size: 5Gi + d3: + device: /dev/sdn + size: 5Gi + d4: + device: /dev/sdo + size: 5Gi + d5: + device: /dev/sdp + size: 5Gi + d6: + device: /dev/sdq + size: 5Gi + d7: + device: /dev/sdr size: 5Gi nodes: node1: diff --git a/k8_cortx_cloud/solution_dummy.yaml b/k8_cortx_cloud/solution_dummy.yaml index 387c8296..e073bb81 100644 --- a/k8_cortx_cloud/solution_dummy.yaml +++ b/k8_cortx_cloud/solution_dummy.yaml @@ -24,7 +24,7 @@ solution: common: cortx_io_svc_ingress: false storage_provisioner_path: /mnt/fs-local-volume - storage: + container_path: local: /etc/cortx shared: /share log: /share/var/log/cortx From 960f03280b4259c5da40f8f5013c17988c611ea4 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 27 Oct 2021 17:44:59 -0600 Subject: [PATCH 05/16] UDX-6040_helm_and_script_updates - fix fs_mount_path did not get assigned correctly --- k8_cortx_cloud/prereq-deploy-cortx-cloud.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh index c027c687..2545e8e0 100755 --- a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh @@ -10,12 +10,6 @@ then exit 1 fi -# Extract storage provisioner path from the "solution.yaml" file -filter='solution.common.storage_provisioner_path' -parse_storage_prov_output=$(parseSolution $filter) -# Get the storage provisioner var from the tuple -fs_mount_path=$(echo $parse_storage_prov_output | cut -f2 -d'>') - if [[ "$disk" == "" ]] then echo "Invalid input paramters" @@ -161,6 +155,12 @@ function prepOpenLdapDeployment() mkdir -p /var/log/3rd-party } +# Extract storage provisioner path from the "solution.yaml" file +filter='solution.common.storage_provisioner_path' +parse_storage_prov_output=$(parseSolution $solution_yaml $filter) +# Get the storage provisioner var from the tuple +fs_mount_path=$(echo $parse_storage_prov_output | cut -f2 -d'>') + cleanupFolders increaseResources prepCortxDeployment From c0d423440dedbe2c1e6d4ed088850801dde85296 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Thu, 28 Oct 2021 12:15:58 -0600 Subject: [PATCH 06/16] Integration updates --- .../deploy-cortx-cloud-3rd-party.sh | 1 + k8_cortx_cloud/deploy-cortx-cloud.sh | 8 +----- k8_cortx_cloud/destroy-cortx-cloud.sh | 7 ----- .../services/cortx-io-svc-ingress.yaml | 27 ------------------- k8_cortx_cloud/shutdown-cortx-cloud.sh | 2 +- k8_cortx_cloud/solution.yaml | 1 - k8_cortx_cloud/solution_dummy.yaml | 1 - k8_cortx_cloud/status-cortx-cloud.sh | 2 +- 8 files changed, 4 insertions(+), 45 deletions(-) delete mode 100644 k8_cortx_cloud/services/cortx-io-svc-ingress.yaml diff --git a/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh b/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh index eb3a0985..ac502429 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh @@ -55,6 +55,7 @@ fi helm install "consul" hashicorp/consul \ --set global.name="consul" \ --set server.storageClass=$storage_class \ + --set ui.enabled=false \ --set server.replicas=$num_consul_replicas printf "######################################################\n" diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index c10237e0..a2c4608a 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -206,6 +206,7 @@ function deployConsul() helm install "consul" hashicorp/consul \ --set global.name="consul" \ --set global.image=$image \ + --set ui.enabled=false \ --set server.storageClass=$storage_class \ --set server.replicas=$num_consul_replicas } @@ -1008,13 +1009,6 @@ function deployCortxServices() printf "# Deploy Services \n" printf "########################################################\n" kubectl apply -f services/cortx-io-svc.yaml --namespace=$namespace - - cortx_io_svc_ingress=$(parseSolution 'solution.common.cortx_io_svc_ingress') - cortx_io_svc_ingress=$(echo $cortx_io_svc_ingress | cut -f2 -d'>') - if [ "$cortx_io_svc_ingress" == "true" ] - then - kubectl apply -f services/cortx-io-svc-ingress.yaml --namespace=$namespace - fi } function deleteCortxProvisioners() diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index 012da60c..e6b5af11 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -94,13 +94,6 @@ function deleteCortxServices() printf "# Delete CORTX Services \n" printf "########################################################\n" kubectl delete service cortx-io-svc --namespace=$namespace - - cortx_io_svc_ingress=$(parseSolution 'solution.common.cortx_io_svc_ingress') - cortx_io_svc_ingress=$(echo $cortx_io_svc_ingress | cut -f2 -d'>') - if [ "$cortx_io_svc_ingress" == "true" ] - then - kubectl delete ingress cortx-io-svc-ingress --namespace=$namespace - fi } function deleteCortxControl() diff --git a/k8_cortx_cloud/services/cortx-io-svc-ingress.yaml b/k8_cortx_cloud/services/cortx-io-svc-ingress.yaml deleted file mode 100644 index a04625ab..00000000 --- a/k8_cortx_cloud/services/cortx-io-svc-ingress.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: cortx-io-svc-ingress -spec: - ingressClassName: nginx - rules: - - host: iam.seagate.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: cortx-io-svc - port: - number: 9080 - - host: s3.seagate.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: cortx-io-svc - port: - number: 80 \ No newline at end of file diff --git a/k8_cortx_cloud/shutdown-cortx-cloud.sh b/k8_cortx_cloud/shutdown-cortx-cloud.sh index 5272924e..83637dc5 100755 --- a/k8_cortx_cloud/shutdown-cortx-cloud.sh +++ b/k8_cortx_cloud/shutdown-cortx-cloud.sh @@ -18,7 +18,7 @@ namespace=$(parseSolution 'solution.namespace') namespace=$(echo $namespace | cut -f2 -d'>') printf "########################################################\n" -printf "# Shutdown CORTX Data \n" +printf "# Shutdown CORTX Control \n" printf "########################################################\n" while IFS= read -r line; do diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index e13f8d4e..d361d1f3 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -22,7 +22,6 @@ solution: gluster: docker.io/gluster/gluster-centos:latest rancher: rancher/local-path-provisioner:v0.0.20 common: - cortx_io_svc_ingress: false storage_provisioner_path: /mnt/fs-local-volume container_path: local: /etc/cortx diff --git a/k8_cortx_cloud/solution_dummy.yaml b/k8_cortx_cloud/solution_dummy.yaml index e073bb81..2265989b 100644 --- a/k8_cortx_cloud/solution_dummy.yaml +++ b/k8_cortx_cloud/solution_dummy.yaml @@ -22,7 +22,6 @@ solution: gluster: docker.io/gluster/gluster-centos:latest rancher: rancher/local-path-provisioner:v0.0.20 common: - cortx_io_svc_ingress: false storage_provisioner_path: /mnt/fs-local-volume container_path: local: /etc/cortx diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh index ecf47a8c..e2bfb858 100755 --- a/k8_cortx_cloud/status-cortx-cloud.sh +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -912,7 +912,7 @@ else fi # Check services cluster IP -num_items=2 +num_items=1 count=0 printf "${INFO}| Checking Services: Cluster IP |${NC}\n" while IFS= read -r line; do From b696d0a63df7126a86f4e1090e2a8bcf63d143b5 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Thu, 28 Oct 2021 14:47:29 -0600 Subject: [PATCH 07/16] Updated replication script --- .../openldap-replication/replication.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh b/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh index 141cd490..db0d1506 100755 --- a/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh +++ b/k8_cortx_cloud/cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh @@ -45,7 +45,7 @@ done for i in "${pod_arr[@]}" do retry_count=1 - while [ ! -z "$(kubectl exec -it $i -- ldapsearch -w $ROOTDN_PASSWORD -x -D cn=admin,cn=config -b cn=config -h localhost 2>/dev/null | grep -o "Can't contact LDAP server (-1)")" ] + while [ ! -z "$(kubectl exec -it $i -- ldapsearch -w $ROOTDN_PASSWORD -x -D cn=admin,cn=config -b cn=config -h localhost 2>/dev/null | grep -o "Can't contact LDAP server (-1)")" ] || [[ "$(kubectl exec -it $i -- ldapsearch -w $ROOTDN_PASSWORD -x -D cn=admin,cn=config -b olcOverlay={1}ppolicy,olcDatabase={2}mdb,cn=config -h localhost 2>/dev/null | grep numEntries:* | awk '{print $3}' | tr -d '\r')" != "1" ]] do if [ $retry_count -eq 11 ] then From 9b4bcf0a24ad112ed407c0389ce9f6bf2f4478be Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Thu, 28 Oct 2021 18:11:57 -0600 Subject: [PATCH 08/16] UDX-6040_helm_and_script_updates - after helm uninstall all cortx, add a wait for all cortx pods to terminate - also add another wait for all 3rd party pods to terminate before deleting all 3rd party pvcs, and pvs --- k8_cortx_cloud/destroy-cortx-cloud.sh | 58 +++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index 012da60c..90262e1e 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -177,6 +177,28 @@ function deleteGlusterfs() done <<< "$(helm ls | grep 'cortx-gluster')" } +function waitForCortxPodsToTerminate() +{ + printf "\nWait for CORTX Pods to terminate" + while true; do + count=0 + cortx_pods="$(kubectl get pods --namespace=$namespace | grep 'cortx\|gluster' 2>&1)" + while IFS= read -r line; do + if [[ "$line" == *"cortx"* || "$line" == *"gluster"* ]]; then + count=$((count+1)) + fi + done <<< "${cortx_pods}" + + if [[ $count -eq 0 ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n\n" +} + function deleteCortxLocalBlockStorage() { printf "######################################################\n" @@ -205,7 +227,7 @@ function deleteCortxPVs() if [[ $line != *"master"* && $line != *"AGE"* ]] then IFS=" " read -r -a pvc_line <<< "$line" - if [[ "${pvc_line[5]}" == *"cortx-"* && "${pvc_line[5]}" != *"openldap"* ]]; then + if [[ "${pvc_line[5]}" == *"cortx-data-fs-local"* || "${pvc_line[5]}" == *"cortx-control-fs-local"* ]]; then printf "Removing ${pvc_line[0]}\n" if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then kubectl patch pv ${pvc_line[0]} -p '{"metadata":{"finalizers":null}}' @@ -314,6 +336,31 @@ function deleteConsul() rm -rf $rancher_prov_path } +function waitFor3rdPartyToTerminate() +{ + printf "\nWait for 3rd party to terminate" + while true; do + count=0 + pods="$(kubectl get pods 2>&1)" + while IFS= read -r line; do + if [[ "$line" == *"kafka"* || \ + "$line" == *"zookeeper"* || \ + "$line" == *"openldap"* || \ + "$line" == *"consul"* ]]; then + count=$((count+1)) + fi + done <<< "${pods}" + + if [[ $count -eq 0 ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n\n" +} + function delete3rdPartyPVCs() { printf "########################################################\n" @@ -366,8 +413,11 @@ function delete3rdPartyPVs() for persistent_volume in $persistent_volumes do printf "Removing $persistent_volume\n" - if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then - kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' + if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then + get_finalizer=$(kubectl get pv gluster-default-volume -o custom-columns=Finalizers:.metadata.finalizers) + if [[ "$get_finalizer" == *"protection"* ]]; then + kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' + fi fi kubectl delete pv $persistent_volume done @@ -426,6 +476,7 @@ deleteCortxServices deleteCortxControl deleteCortxProvisioners deleteGlusterfs +waitForCortxPodsToTerminate deleteCortxLocalBlockStorage deleteCortxPVs deleteCortxConfigmap @@ -437,6 +488,7 @@ deleteKafkaZookeper deleteOpenLdap deleteSecrets deleteConsul +waitFor3rdPartyToTerminate delete3rdPartyPVCs delete3rdPartyPVs From 6fafb80d903ea8cd7aa2e5c85d57b940d96fb6cd Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 29 Oct 2021 09:26:04 -0600 Subject: [PATCH 09/16] Integration updates * Adoption of new symas OpenLDAP image in cortx-services script [UDX-6507] * Skip Replication in OpenLDAP for single node setup [UDX-6506] --- k8_cortx_cloud/deploy-cortx-cloud.sh | 4 +++- k8_cortx_cloud/solution.yaml | 2 +- k8_cortx_cloud/solution_dummy.yaml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index a2c4608a..353e79ac 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -256,7 +256,9 @@ function deployOpenLDAP() printf "Setup OpenLDAP replication \n" printf "===========================================================\n" # Run replication script - ./cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh --rootdnpassword $openldap_password + if [[ $num_openldap_replicas -gt 1 ]]; then + ./cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh --rootdnpassword $openldap_password + fi } function deployZookeeper() diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index d361d1f3..bfdf127f 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -15,7 +15,7 @@ solution: cortxcontrol: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci cortxdataprov: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci cortxdata: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci - openldap: ghcr.io/seagate/symas-openldap:standalone + openldap: ghcr.io/seagate/symas-openldap:2.4.58 consul: hashicorp/consul:1.10.0 kafka: bitnami/kafka:3.0.0-debian-10-r7 zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 diff --git a/k8_cortx_cloud/solution_dummy.yaml b/k8_cortx_cloud/solution_dummy.yaml index 2265989b..486c98d0 100644 --- a/k8_cortx_cloud/solution_dummy.yaml +++ b/k8_cortx_cloud/solution_dummy.yaml @@ -15,7 +15,7 @@ solution: cortxcontrol: centos:7 cortxdataprov: centos:7 cortxdata: centos:7 - openldap: ghcr.io/seagate/symas-openldap:standalone + openldap: ghcr.io/seagate/symas-openldap:2.4.58 consul: hashicorp/consul:1.10.0 kafka: bitnami/kafka:3.0.0-debian-10-r7 zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 From c16c6b445090684ae05c19dc992ae04149ead9b8 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Fri, 29 Oct 2021 11:09:30 -0600 Subject: [PATCH 10/16] UDX-6040_helm_and_script_updates - fix typo in delete3rdPartyPVs function --- k8_cortx_cloud/destroy-cortx-cloud.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index a06b71b2..b37e64d8 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -395,7 +395,7 @@ function delete3rdPartyPVs() do printf "Removing $persistent_volume\n" if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then - kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' + kubectl patch pv $persistent_volume -p '{"metadata":{"finalizers":null}}' fi kubectl delete pv $persistent_volume done @@ -406,11 +406,8 @@ function delete3rdPartyPVs() for persistent_volume in $persistent_volumes do printf "Removing $persistent_volume\n" - if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then - get_finalizer=$(kubectl get pv gluster-default-volume -o custom-columns=Finalizers:.metadata.finalizers) - if [[ "$get_finalizer" == *"protection"* ]]; then - kubectl patch pvc $persistent_volume -p '{"metadata":{"finalizers":null}}' - fi + if [[ "$force_delete" == "--force" || "$force_delete" == "-f" ]]; then + kubectl patch pv $persistent_volume -p '{"metadata":{"finalizers":null}}' fi kubectl delete pv $persistent_volume done From 1bc6dc7ef21db99addd123b21af5bf2ed6c2aff9 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Fri, 29 Oct 2021 14:22:14 -0600 Subject: [PATCH 11/16] UDX-6040_helm_and_script_updates - add a wait for glusterfs to terminate - reorder the functions in the destroy script. Move deleteGlusterfs after waitForCortxPodsToTerminate. --- k8_cortx_cloud/destroy-cortx-cloud.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index b37e64d8..23dd2c8b 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -168,6 +168,25 @@ function deleteGlusterfs() IFS=" " read -r -a my_array <<< "$line" helm uninstall ${my_array[0]} done <<< "$(helm ls | grep 'cortx-gluster')" + + printf "\nWait for GlusterFS to terminate" + while true; do + count=0 + glusterfs="$(kubectl get pods --namespace=$namespace | grep 'gluster' 2>&1)" + while IFS= read -r line; do + if [[ "$line" == *"gluster"* ]]; then + count=$((count+1)) + fi + done <<< "${glusterfs}" + + if [[ $count -eq 0 ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n\n" } function waitForCortxPodsToTerminate() @@ -175,9 +194,9 @@ function waitForCortxPodsToTerminate() printf "\nWait for CORTX Pods to terminate" while true; do count=0 - cortx_pods="$(kubectl get pods --namespace=$namespace | grep 'cortx\|gluster' 2>&1)" + cortx_pods="$(kubectl get pods --namespace=$namespace | grep 'cortx' 2>&1)" while IFS= read -r line; do - if [[ "$line" == *"cortx"* || "$line" == *"gluster"* ]]; then + if [[ "$line" == *"cortx"* ]]; then count=$((count+1)) fi done <<< "${cortx_pods}" @@ -465,8 +484,8 @@ deleteCortxData deleteCortxServices deleteCortxControl deleteCortxProvisioners -deleteGlusterfs waitForCortxPodsToTerminate +deleteGlusterfs deleteCortxLocalBlockStorage deleteCortxPVs deleteCortxConfigmap From 4ba184fce1a5262854cddfe2baba737a6ba52be4 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 29 Oct 2021 15:39:38 -0600 Subject: [PATCH 12/16] Integration Updates * Move storage provisioner destroy to clean section --- k8_cortx_cloud/destroy-cortx-cloud.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index 23dd2c8b..9e89cd3f 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -341,11 +341,6 @@ function deleteConsul() printf "# Delete Consul #\n" printf "########################################################\n" helm delete consul - - rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" - rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" - kubectl delete -f $rancher_prov_file - rm -rf $rancher_prov_path } function waitFor3rdPartyToTerminate() @@ -433,6 +428,14 @@ function delete3rdPartyPVs() fi } +function deleteStorageProvisioner() +{ + rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" + rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" + kubectl delete -f $rancher_prov_file + rm -rf $rancher_prov_path +} + function helmChartCleanup() { print_header=true @@ -504,6 +507,7 @@ delete3rdPartyPVs ############################################################# # Clean up ############################################################# +deleteStorageProvisioner helmChartCleanup deleteCortxNamespace cleanup \ No newline at end of file From 6e289ac0c2f243e072098e5974c5461e3405dbf2 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Mon, 1 Nov 2021 12:18:41 -0600 Subject: [PATCH 13/16] UDX-6040_helm_and_script_updates - Disable glusterfs health check. If this is not disabled and this check fails, the file system exported by the brick is not usable anymore and the brick process (glusterfsd) logs a warning and exits --- k8_cortx_cloud/deploy-cortx-cloud.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 353e79ac..dd89890a 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -475,6 +475,10 @@ function deployCortxGlusterFS() kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster volume create $gluster_vol $first_gluster_ip:$gluster_folder force fi + # Disable gluster health check. If this is not disabled and this check fails, the file system exported by + # the brick is not usable anymore and the brick process (glusterfsd) logs a warning and exits + kubectl exec -i $first_gluster_node_name --namespace=$namespace -- gluster volume set $gluster_vol storage.health-check-interval 0 + # Start gluster volume echo y | kubectl exec -i $first_gluster_node_name --namespace=$namespace --namespace=$namespace -- gluster volume start $gluster_vol } From 5bd1085577ba1741f62075bcfafad30ff4a3acdf Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Mon, 1 Nov 2021 13:17:42 -0600 Subject: [PATCH 14/16] UDX-6052 Integration updates *Removed 3rd party scripts and readme * Fixed kafka sequencing and images pulls for kafka and zookeeper --- k8_cortx_cloud/README_for3rdPartyOnly.txt | 40 ----- .../deploy-cortx-cloud-3rd-party.sh | 157 ------------------ k8_cortx_cloud/deploy-cortx-cloud.sh | 30 ++++ .../destroy-cortx-cloud-3rd-party.sh | 65 -------- k8_cortx_cloud/solution.yaml | 4 +- k8_cortx_cloud/solution_dummy.yaml | 4 +- 6 files changed, 34 insertions(+), 266 deletions(-) delete mode 100644 k8_cortx_cloud/README_for3rdPartyOnly.txt delete mode 100755 k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh delete mode 100755 k8_cortx_cloud/destroy-cortx-cloud-3rd-party.sh diff --git a/k8_cortx_cloud/README_for3rdPartyOnly.txt b/k8_cortx_cloud/README_for3rdPartyOnly.txt deleted file mode 100644 index 4f875af3..00000000 --- a/k8_cortx_cloud/README_for3rdPartyOnly.txt +++ /dev/null @@ -1,40 +0,0 @@ -############################################### -# Rancher Local Path Provisioner Requirements # -############################################### -1. Create a directory for Rancher local path provisioner on each worker node, and - untained master node that allows scheduling: -mkdir -p /mnt/fs-local-volume/local-path-provisioner -mkfs.ext4 -mount -t ext4 /mnt/fs-local-volume - -Example: -mkfs.ext4 /dev/sdd -mount -t ext4 /dev/sdd /mnt/fs-local-volume - -Rancher Local Path location on worker node: -/mnt/fs-local-volume/local-path-provisioner/pvc-_default_cortx-fs-local-pvc- - -Rancher Local Path location in all Pod containers (CORTX Provisioner, Data, -Control, and Support): -/data - -Note: if there's no disk partition on all worker nodes then just create folder -"/mnt/fs-local-volume/local-path-provisioner" with the exact same path on all -worker nodes. - -############################################### -# OpenLDAP Requirements # -############################################### -1. On each worker node perform the following: -mkdir -p /etc/3rd-party/openldap -mkdir -p /var/data/3rd-party -mkdir -p /var/log/3rd-party - -############################################### -# Deploy and destroy CORTX cloud 3rd party # -############################################### -1. Deploy 3rd party: -./deploy-cortx-cloud-3rd-party.sh - -2. Destroy 3rd party: -./destroy-cortx-cloud-3rd-party.sh \ No newline at end of file diff --git a/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh b/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh deleted file mode 100755 index ac502429..00000000 --- a/k8_cortx_cloud/deploy-cortx-cloud-3rd-party.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash - -openldap_password=${1:-seagate1} - -storage_class='local-path' -printf "Default storage class: $storage_class\n" - -# Delete old "node-list-info.txt" file -find $(pwd)/cortx-cloud-3rd-party-pkg/openldap -name "node-list-info*" -delete - -max_openldap_inst=3 # Default max openldap instances -max_consul_inst=3 -max_kafka_inst=3 -num_openldap_replicas=0 # Default the number of actual openldap instances -num_worker_nodes=0 -while IFS= read -r line; do - IFS=" " read -r -a node_name <<< "$line" - if [[ "$node_name" != "NAME" ]]; then - output=$(kubectl describe nodes $node_name | grep Taints | grep NoSchedule) - if [[ "$output" == "" ]]; then - node_list_str="$num_worker_nodes $node_name" - num_worker_nodes=$((num_worker_nodes+1)) - - if [[ "$num_worker_nodes" -le "$max_openldap_inst" ]]; then - num_openldap_replicas=$num_worker_nodes - node_list_info_path=$(pwd)/cortx-cloud-3rd-party-pkg/openldap/node-list-info.txt - if [[ -s $node_list_info_path ]]; then - printf "\n" >> $node_list_info_path - fi - printf "$node_list_str" >> $node_list_info_path - fi - fi - fi -done <<< "$(kubectl get nodes)" -printf "Number of worker nodes detected: $num_worker_nodes\n" - -printf "######################################################\n" -printf "# Deploy Consul \n" -printf "######################################################\n" -num_consul_replicas=$num_worker_nodes -if [[ "$num_worker_nodes" -gt "$max_consul_inst" ]]; then - num_consul_replicas=$max_consul_inst -fi - -# Add the HashiCorp Helm Repository: -helm repo add hashicorp https://helm.releases.hashicorp.com -if [[ $storage_class == "local-path" ]] -then - printf "Install Rancher Local Path Provisioner" - # Install Rancher provisioner - # kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml - kubectl create -f cortx-cloud-3rd-party-pkg/local-path-storage.yaml -fi - -helm install "consul" hashicorp/consul \ - --set global.name="consul" \ - --set server.storageClass=$storage_class \ - --set ui.enabled=false \ - --set server.replicas=$num_consul_replicas - -printf "######################################################\n" -printf "# Deploy openLDAP \n" -printf "######################################################\n" -helm install "openldap" cortx-cloud-3rd-party-pkg/openldap \ - --set openldap.servicename="openldap-svc" \ - --set openldap.storageclass="openldap-local-storage" \ - --set openldap.storagesize="5Gi" \ - --set openldap.nodelistinfo="node-list-info.txt" \ - --set openldap.numreplicas=$num_openldap_replicas \ - --set openldap.password=$openldap_password - -# Wait for all openLDAP pods to be ready -printf "\nWait for openLDAP PODs to be ready" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[2]}" - if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then - break - fi - count=$((count+1)) - done <<< "$(kubectl get pods -A | grep 'openldap')" - - if [[ $count -eq $num_openldap_replicas ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n\n" - -printf "===========================================================\n" -printf "Setup OpenLDAP replication \n" -printf "===========================================================\n" -# Run replication script -./cortx-cloud-3rd-party-pkg/openldap-replication/replication.sh --rootdnpassword $openldap_password - -printf "######################################################\n" -printf "# Deploy Zookeeper \n" -printf "######################################################\n" -num_kafka_replicas=$num_worker_nodes -if [[ "$num_worker_nodes" -gt "$max_kafka_inst" ]]; then - num_kafka_replicas=$max_kafka_inst -fi - -# Add Zookeeper and Kafka Repository -helm repo add bitnami https://charts.bitnami.com/bitnami - -helm install zookeeper bitnami/zookeeper \ - --set replicaCount=$num_kafka_replicas \ - --set auth.enabled=false \ - --set allowAnonymousLogin=true \ - --set global.storageClass=$storage_class - -printf "######################################################\n" -printf "# Deploy Kafka \n" -printf "######################################################\n" -helm install kafka bitnami/kafka \ - --set zookeeper.enabled=false \ - --set replicaCount=$num_kafka_replicas \ - --set externalZookeeper.servers=zookeeper.default.svc.cluster.local \ - --set global.storageClass=$storage_class \ - --set defaultReplicationFactor=$num_kafka_replicas \ - --set offsetTopicReplicationFactor=$num_kafka_replicas \ - --set transactionStateLogReplicationFactor=$num_kafka_replicas \ - --set auth.enabled=false \ - --set allowAnonymousLogin=true \ - --set deleteTopicEnable=true \ - --set transactionStateLogMinIsr=2 - -printf "\nWait for CORTX 3rd party to be ready" -while true; do - count=0 - while IFS= read -r line; do - IFS=" " read -r -a pod_status <<< "$line" - IFS="/" read -r -a ready_status <<< "${pod_status[1]}" - if [[ "${pod_status[2]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then - count=$((count+1)) - break - fi - done <<< "$(kubectl get pods --namespace=default | grep 'consul\|kafka\|openldap\|zookeeper')" - - if [[ $count -eq 0 ]]; then - break - else - printf "." - fi - sleep 1s -done -printf "\n" - -################################################################# -# Delete files that contain node info -################################################################# -find $(pwd)/cortx-cloud-3rd-party-pkg/openldap -name "node-list-info*" -delete \ No newline at end of file diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 353e79ac..f46d63cd 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -269,11 +269,36 @@ function deployZookeeper() # Add Zookeeper and Kafka Repository helm repo add bitnami https://charts.bitnami.com/bitnami + image=$(parseSolution 'solution.images.zookeeper') + image=$(echo $image | cut -f2 -d'>') + helm install zookeeper bitnami/zookeeper \ + --set image.tag=$image \ --set replicaCount=$num_kafka_replicas \ --set auth.enabled=false \ --set allowAnonymousLogin=true \ --set global.storageClass=$storage_class + + printf "\nWait for Zookeeper to be ready before starting kafka" + while true; do + count=0 + while IFS= read -r line; do + IFS=" " read -r -a pod_status <<< "$line" + IFS="/" read -r -a ready_status <<< "${pod_status[2]}" + if [[ "${pod_status[3]}" != "Running" || "${ready_status[0]}" != "${ready_status[1]}" ]]; then + count=$((count+1)) + break + fi + done <<< "$(kubectl get pods -A | grep 'zookeeper')" + + if [[ $count -eq 0 ]]; then + break + else + printf "." + fi + sleep 1s + done + printf "\n\n" } function deployKafka() @@ -281,8 +306,13 @@ function deployKafka() printf "######################################################\n" printf "# Deploy Kafka \n" printf "######################################################\n" + + image=$(parseSolution 'solution.images.kafka') + image=$(echo $image | cut -f2 -d'>') + helm install kafka bitnami/kafka \ --set zookeeper.enabled=false \ + --set image.tag=$image \ --set replicaCount=$num_kafka_replicas \ --set externalZookeeper.servers=zookeeper.default.svc.cluster.local \ --set global.storageClass=$storage_class \ diff --git a/k8_cortx_cloud/destroy-cortx-cloud-3rd-party.sh b/k8_cortx_cloud/destroy-cortx-cloud-3rd-party.sh deleted file mode 100755 index cbc9f1e3..00000000 --- a/k8_cortx_cloud/destroy-cortx-cloud-3rd-party.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -PVC_CONSUL_FILTER="data-default-consul" -PVC_KAFKA_FILTER="kafka" -PVC_ZOOKEEPER_FILTER="zookeeper" -PV_FILTER="pvc" -OPENLDAP_PVC="openldap-data" - -printf "###################################\n" -printf "# Delete Kafka #\n" -printf "###################################\n" -helm uninstall kafka - -printf "###################################\n" -printf "# Delete Zookeeper #\n" -printf "###################################\n" -helm uninstall zookeeper - -printf "########################################################\n" -printf "# Delete openLDAP #\n" -printf "########################################################\n" -openldap_array=[] -count=0 -while IFS= read -r line; do - IFS=" " read -r -a my_array <<< "$line" - openldap_array[count]="${my_array[1]}" - count=$((count+1)) -done <<< "$(kubectl get pods -A | grep 'openldap-')" - -for openldap_pod_name in "${openldap_array[@]}" -do - kubectl exec -ti $openldap_pod_name --namespace="default" -- bash -c \ - 'rm -rf /etc/3rd-party/* /var/data/3rd-party/* /var/log/3rd-party/*' -done - -helm uninstall "openldap" - -printf "###################################\n" -printf "# Delete Consul #\n" -printf "###################################\n" -helm delete consul -# kubectl delete -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml -kubectl delete -f cortx-cloud-3rd-party-pkg/local-path-storage.yaml - -printf "###################################\n" -printf "# Delete Persistent Volume Claims #\n" -printf "###################################\n" -VOLUME_CLAIMS=$(kubectl get pvc | grep -E "$PVC_CONSUL_FILTER|$PVC_KAFKA_FILTER|$PVC_ZOOKEEPER_FILTER|$OPENLDAP_PVC|cortx" | cut -f1 -d " ") -echo $VOLUME_CLAIMS -for VOLUME_CLAIM in $VOLUME_CLAIMS -do - printf "Removing $VOLUME_CLAIM\n" - kubectl delete pvc $VOLUME_CLAIM -done - -printf "###################################\n" -printf "# Delete Persistent Volumes #\n" -printf "###################################\n" -PERSISTENT_VOLUMES=$(kubectl get pv | grep -E "$PVC_CONSUL_FILTER|$PVC_KAFKA_FILTER|$PVC_ZOOKEEPER_FILTER|cortx" | cut -f1 -d " ") -echo $PERSISTENT_VOLUMES -for PERSISTENT_VOLUME in $PERSISTENT_VOLUMES -do - printf "Removing $PERSISTENT_VOLUME\n" - kubectl delete pv $PERSISTENT_VOLUME -done diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index bfdf127f..7a984c91 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -17,8 +17,8 @@ solution: cortxdata: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci openldap: ghcr.io/seagate/symas-openldap:2.4.58 consul: hashicorp/consul:1.10.0 - kafka: bitnami/kafka:3.0.0-debian-10-r7 - zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 + kafka: 3.0.0-debian-10-r7 + zookeeper: 3.7.0-debian-10-r182 gluster: docker.io/gluster/gluster-centos:latest rancher: rancher/local-path-provisioner:v0.0.20 common: diff --git a/k8_cortx_cloud/solution_dummy.yaml b/k8_cortx_cloud/solution_dummy.yaml index 486c98d0..13be363c 100644 --- a/k8_cortx_cloud/solution_dummy.yaml +++ b/k8_cortx_cloud/solution_dummy.yaml @@ -17,8 +17,8 @@ solution: cortxdata: centos:7 openldap: ghcr.io/seagate/symas-openldap:2.4.58 consul: hashicorp/consul:1.10.0 - kafka: bitnami/kafka:3.0.0-debian-10-r7 - zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 + kafka: 3.0.0-debian-10-r7 + zookeeper: 3.7.0-debian-10-r182 gluster: docker.io/gluster/gluster-centos:latest rancher: rancher/local-path-provisioner:v0.0.20 common: From 1cce07aceeddec75604aa2451aecfe92399df625 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Tue, 2 Nov 2021 09:47:24 -0600 Subject: [PATCH 15/16] UDX-6522 Added max_start_timeout to solution.yaml --- .../cortx-configmap/templates/config-template.yaml | 1 + k8_cortx_cloud/deploy-cortx-cloud.sh | 1 + k8_cortx_cloud/solution.yaml | 1 + k8_cortx_cloud/solution_dummy.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-configmap/templates/config-template.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-configmap/templates/config-template.yaml index f96d0236..6b22b812 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-configmap/templates/config-template.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-configmap/templates/config-template.yaml @@ -51,6 +51,7 @@ cortx: - http://<<.Values.cortx.io.svc>>:28049 service_instances: <<.Values.cortx.num_s3_inst>> io_max_units: 8 + max_start_timeout: <<.Values.cortx.max_start_timeout>> auth_admin: sgiamadmin auth_secret: s3_auth_admin_secret motr: diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 665bb7a0..5313c7ff 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -570,6 +570,7 @@ function deployCortxConfigMap() ./parse_scripts/subst.sh $new_gen_file "cortx.external.consul.endpoints" $consul_endpoint ./parse_scripts/subst.sh $new_gen_file "cortx.io.svc" "cortx-io-svc" ./parse_scripts/subst.sh $new_gen_file "cortx.num_s3_inst" $(extractBlock 'solution.common.s3.num_inst') + ./parse_scripts/subst.sh $new_gen_file "cortx.max_start_timeout" $(extractBlock 'solution.common.s3.max_start_timeout') ./parse_scripts/subst.sh $new_gen_file "cortx.num_motr_inst" $(extractBlock 'solution.common.motr.num_client_inst') ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.local" $local_storage ./parse_scripts/subst.sh $new_gen_file "cortx.common.storage.shared" $shared_storage diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index 7a984c91..4d593e9b 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -30,6 +30,7 @@ solution: s3: num_inst: 2 start_port_num: 28051 + max_start_timeout: 240 motr: num_client_inst: 0 start_port_num: 29000 diff --git a/k8_cortx_cloud/solution_dummy.yaml b/k8_cortx_cloud/solution_dummy.yaml index 13be363c..1a7c4098 100644 --- a/k8_cortx_cloud/solution_dummy.yaml +++ b/k8_cortx_cloud/solution_dummy.yaml @@ -30,6 +30,7 @@ solution: s3: num_inst: 2 start_port_num: 28051 + max_start_timeout: 240 motr: num_client_inst: 0 start_port_num: 29000 From 4f4d8adfb8368b572276e0cb324a7805b5e135d8 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Tue, 2 Nov 2021 10:39:53 -0600 Subject: [PATCH 16/16] Updated version file --- k8_cortx_cloud/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8_cortx_cloud/VERSION b/k8_cortx_cloud/VERSION index 5b9168fe..563fffcf 100644 --- a/k8_cortx_cloud/VERSION +++ b/k8_cortx_cloud/VERSION @@ -1 +1 @@ -VERSION v0.0.11 \ No newline at end of file +VERSION v0.0.12 \ No newline at end of file