From 555010ddb181be7c6cca4575ecd8f80158106814 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:30:16 -0600 Subject: [PATCH 01/12] UDX-6308_switch_over_from_dummy_to_CORTX_containers - Switch all dummy containers over to cortx containers --- .../templates/cortx-control-prov-pod.yaml | 2 +- .../templates/cortx-control-pod.yaml | 39 +++++++++++++- .../templates/cortx-data-prov-pod.yaml | 2 +- .../cortx-data/templates/cortx-data-pod.yaml | 51 +++++++++++++++---- k8_cortx_cloud/solution.yaml | 16 +++--- 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml index 82af16ac..c7e1cef5 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml @@ -36,7 +36,7 @@ spec: - /bin/sh args: - -c - - sleep $(shuf -i 5-10 -n 1)s + - /opt/seagate/cortx/provisioner/bin/cortx_deploy -f /etc/cortx/solution -c yaml:///etc/cortx/cluster.conf ; volumeMounts: - name: {{ .Values.cortxcontrolprov.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrolprov.cfgmap.mountpath }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml index 54f22dc0..d48cfbb3 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml @@ -61,10 +61,41 @@ spec: env: - name: UDS_CLOUD_CONTAINER_NAME value: {{ .Values.cortxcontrol.name }} + - name: cortx-fsm-motr + image: {{ .Values.cortxcontrol.image }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/motr/bin/motr_setup start --services fsm --config yaml:///etc/cortx/cluster.conf; + volumeMounts: + - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} + mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} + - name: {{ .Values.cortxcontrol.sslcfgmap.volmountname }} + mountPath: {{ .Values.cortxcontrol.sslcfgmap.mountpath }} + - name: {{ .Values.cortxcontrol.machineid.volmountname }} + mountPath: {{ .Values.cortxcontrol.machineid.mountpath }} + - name: {{ .Values.cortxgluster.pv.name }} + mountPath: {{ .Values.cortxgluster.pv.mountpath }} + - name: local-path-pv + mountPath: {{ .Values.cortxcontrol.localpathpvc.mountpath }} + {{- range .Files.Lines .Values.cortxcontrol.secretinfo }} + - name: {{ printf "%s" . }} + mountPath: /etc/cortx/solution/secret + readOnly: true + {{- end }} + env: + - name: UDS_CLOUD_CONTAINER_NAME + value: {{ .Values.cortxcontrol.name }} - name: cortx-bg-producer image: {{ .Values.cortxcontrol.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/s3/bin/s3_start --service s3bgproducer -c yaml:///etc/cortx/cluster.conf; volumeMounts: - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} @@ -87,7 +118,11 @@ spec: - name: cortx-csm-agent image: {{ .Values.cortxcontrol.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /usr/bin/csm_agent start -c yaml:///etc/cortx/cluster.conf; volumeMounts: - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml index 7e434582..ec04008f 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml @@ -43,7 +43,7 @@ spec: - /bin/sh args: - -c - - sleep $(shuf -i 5-10 -n 1)s + - /opt/seagate/cortx/provisioner/bin/cortx_deploy -f /etc/cortx/solution -c yaml:///etc/cortx/cluster.conf ; volumeDevices: {{- range .Files.Lines .Values.cortxdataprov.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml index e6602119..c1567fc1 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml @@ -49,7 +49,11 @@ spec: - name: cortx-s3-haproxy image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/s3/bin/s3_start --service haproxy -c yaml:///etc/cortx/cluster.conf; volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -83,7 +87,11 @@ spec: - name: cortx-s3-auth-server image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/s3/bin/s3_start --service s3authserver -c yaml:///etc/cortx/cluster.conf; volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -117,8 +125,11 @@ spec: value: {{ $.Values.cortxdata.name }} - name: S3_INDEX value: {{ printf "%d" (add 1 $i) | quote }} - command: ["/bin/sh", "-c"] - args: ["echo $S3_INDEX > /root/testing; /bin/sleep 3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/s3/bin/s3_start --service s3server --index $S3_INDEX -c yaml:///etc/cortx/cluster.conf; volumeMounts: - name: {{ $.Values.cortxdata.cfgmap.volmountname }} mountPath: {{ $.Values.cortxdata.cfgmap.mountpath }} @@ -141,7 +152,11 @@ spec: - name: cortx-motr-hax image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/hare/bin/hare_setup start --config yaml:///etc/cortx/cluster.conf; volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -199,7 +214,11 @@ spec: - name: cortx-motr-confd image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/motr/bin/motr_setup start --services confd --config yaml:///etc/cortx/cluster.conf; volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -228,7 +247,16 @@ spec: - name: {{ printf "cortx-motr-ios-%03d" (add 1 $i) }} image: {{ $.Values.cortxdata.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + env: + - name: UDS_CLOUD_CONTAINER_NAME + value: {{ $.Values.cortxdata.name }} + - name: IO_INDEX + value: {{ printf "%d" (add 1 $i) | quote }} + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/motr/bin/motr_setup start --services ioservice --idx $IO_INDEX --config yaml:///etc/cortx/cluster.conf; volumeDevices: {{- range $.Files.Lines $.Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -250,16 +278,17 @@ spec: mountPath: /etc/cortx/solution/secret readOnly: true {{- end }} - env: - - name: UDS_CLOUD_CONTAINER_NAME - value: {{ $.Values.cortxdata.name }} ports: - containerPort: {{ printf "%d" (add $i $.Values.cortxdata.motr.startportnum) }} {{- end }} - name: cortx-s3-bg-consumer image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent - command: ["/bin/sleep", "3650d"] + command: + - /bin/sh + args: + - -c + - /opt/seagate/cortx/s3/bin/s3_start --service s3bgconsumer -c yaml:///etc/cortx/cluster.conf; volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index 7aa0ac2b..139f78a3 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -3,7 +3,7 @@ solution: secrets: name: cortx-secret content: - openldap_admin_secret: seagate2 + openldap_admin_secret: seagate1 kafka_admin_secret: Seagate@123 consul_admin_secret: Seagate@123 common_admin_secret: Seagate@123 @@ -11,11 +11,11 @@ solution: csm_auth_admin_secret: seagate2 csm_mgmt_admin_secret: Cortxadmin@123 images: - cortxcontrolprov: centos:7 - cortxcontrol: centos:7 - cortxdataprov: centos:7 - cortxdata: centos:7 - cortxsupport: centos:7 + cortxcontrolprov: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxcontrol: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxdataprov: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxdata: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxsupport: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci 3rdparty: openldap: password: seagate1 @@ -29,12 +29,12 @@ solution: num_inst: 2 start_port_num: 28051 motr: - num_client_inst: 1 + num_client_inst: 0 start_port_num: 29000 storage_sets: name: storage-set-1 durability: - sns: 8+7+0 + sns: 1+0+0 dix: 1+0+0 storage: cvg1: From ba9e53a21504b6e84283f22a9994407cc8f455b2 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:43:59 -0600 Subject: [PATCH 02/12] UDX-6308_switch_over_from_dummy_to_CORTX_containers - Revert a change accidentally made in the solution file. --- k8_cortx_cloud/solution.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/k8_cortx_cloud/solution.yaml b/k8_cortx_cloud/solution.yaml index 3a1a2227..7e4b34ea 100644 --- a/k8_cortx_cloud/solution.yaml +++ b/k8_cortx_cloud/solution.yaml @@ -73,3 +73,11 @@ solution: name: node-2 devices: system: /mnt/fs-local-volume/ + node3: + name: node-3 + devices: + system: /mnt/fs-local-volume/ + node4: + name: node-4 + devices: + system: /mnt/fs-local-volume/ \ No newline at end of file From 0707ed60419cb4c13bc49fa9663b5c5e238875d6 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Wed, 20 Oct 2021 17:12:54 -0600 Subject: [PATCH 03/12] UDX-6308_switch_over_from_dummy_to_CORTX_containers - update README.txt --- k8_cortx_cloud/README.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/k8_cortx_cloud/README.txt b/k8_cortx_cloud/README.txt index 37f99992..e4c24ece 100644 --- a/k8_cortx_cloud/README.txt +++ b/k8_cortx_cloud/README.txt @@ -15,13 +15,19 @@ has to mount it manually as instructed below. ############################################### # Run prerequisite deployment script # ############################################### -1. Copy "prereq-deploy-cortx-cloud.sh" script to all worker nodes, and untainted master - node that allows scheduling: +1. Copy "prereq-deploy-cortx-cloud.sh" script, "solution.yaml" file, and + "parse_scripts" folder to all worker nodes: -scp prereq-deploy-cortx-cloud.sh root@: +scp prereq-deploy-cortx-cloud.sh @: +scp solution.yaml @: +mkdir -p /parse_scripts +scp -r ./parse_scripts/* @:/parse_scripts Example: -scp prereq-deploy-cortx-cloud.sh root@192.168.1.1:/home/ +scp prereq-deploy-cortx-cloud.sh @192.168.1.1:/home/ +scp solution.yaml @192.168.1.1:/home/ +mkdir -p /home/parse_scripts +scp -r ./parse_scripts/* @192.168.1.1:/home/parse_scripts 2. Run prerequisite script on all worker nodes, and untainted master node that allows scheduling. "" is a required input to run this script. This disk From 6fb0b6257c161833c61dfb6dc3593262af23bcbf Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:59:27 -0600 Subject: [PATCH 04/12] UDX-6308_switch_over_from_dummy_to_CORTX_containers - add the ability to switch between dummy containers (centos:7) and cortx containers. Change 'solution.images.cortx*" in the solution file to 'centos:7' or 'ghcr.io/seagate/cortx-all:'. --- .../templates/cortx-control-prov-pod.yaml | 6 ++++ .../templates/cortx-control-pod.yaml | 12 ++++++++ .../templates/cortx-data-prov-pod.yaml | 6 ++++ .../cortx-data/templates/cortx-data-pod.yaml | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml index c7e1cef5..ac4ec182 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control-provisioner/templates/cortx-control-prov-pod.yaml @@ -34,9 +34,15 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/sh + {{- if eq .Values.cortxcontrolprov.image "centos:7" }} + args: + - -c + - sleep $(shuf -i 5-10 -n 1)s + {{- else }} args: - -c - /opt/seagate/cortx/provisioner/bin/cortx_deploy -f /etc/cortx/solution -c yaml:///etc/cortx/cluster.conf ; + {{- end }} volumeMounts: - name: {{ .Values.cortxcontrolprov.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrolprov.cfgmap.mountpath }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml index d48cfbb3..efa2ec88 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml @@ -64,11 +64,15 @@ spec: - name: cortx-fsm-motr image: {{ .Values.cortxcontrol.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxcontrol.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/motr/bin/motr_setup start --services fsm --config yaml:///etc/cortx/cluster.conf; + {{- end }} volumeMounts: - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} @@ -91,11 +95,15 @@ spec: - name: cortx-bg-producer image: {{ .Values.cortxcontrol.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxcontrol.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/s3/bin/s3_start --service s3bgproducer -c yaml:///etc/cortx/cluster.conf; + {{- end }} volumeMounts: - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} @@ -118,11 +126,15 @@ spec: - name: cortx-csm-agent image: {{ .Values.cortxcontrol.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxcontrol.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /usr/bin/csm_agent start -c yaml:///etc/cortx/cluster.conf; + {{- end }} volumeMounts: - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml index ec04008f..8e0c3076 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data-provisioner/templates/cortx-data-prov-pod.yaml @@ -41,9 +41,15 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/sh + {{- if eq .Values.cortxdataprov.image "centos:7" }} + args: + - -c + - sleep $(shuf -i 5-10 -n 1)s + {{- else }} args: - -c - /opt/seagate/cortx/provisioner/bin/cortx_deploy -f /etc/cortx/solution -c yaml:///etc/cortx/cluster.conf ; + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdataprov.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml index c1567fc1..f8a91064 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml @@ -49,11 +49,15 @@ spec: - name: cortx-s3-haproxy image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/s3/bin/s3_start --service haproxy -c yaml:///etc/cortx/cluster.conf; + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -87,11 +91,15 @@ spec: - name: cortx-s3-auth-server image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/s3/bin/s3_start --service s3authserver -c yaml:///etc/cortx/cluster.conf; + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -125,11 +133,15 @@ spec: value: {{ $.Values.cortxdata.name }} - name: S3_INDEX value: {{ printf "%d" (add 1 $i) | quote }} + {{- if eq $.Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/s3/bin/s3_start --service s3server --index $S3_INDEX -c yaml:///etc/cortx/cluster.conf; + {{- end }} volumeMounts: - name: {{ $.Values.cortxdata.cfgmap.volmountname }} mountPath: {{ $.Values.cortxdata.cfgmap.mountpath }} @@ -152,11 +164,15 @@ spec: - name: cortx-motr-hax image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/hare/bin/hare_setup start --config yaml:///etc/cortx/cluster.conf; + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -214,11 +230,15 @@ spec: - name: cortx-motr-confd image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/motr/bin/motr_setup start --services confd --config yaml:///etc/cortx/cluster.conf; + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -252,11 +272,15 @@ spec: value: {{ $.Values.cortxdata.name }} - name: IO_INDEX value: {{ printf "%d" (add 1 $i) | quote }} + {{- if eq $.Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/motr/bin/motr_setup start --services ioservice --idx $IO_INDEX --config yaml:///etc/cortx/cluster.conf; + {{- end }} volumeDevices: {{- range $.Files.Lines $.Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -284,11 +308,15 @@ spec: - name: cortx-s3-bg-consumer image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: - /bin/sh args: - -c - /opt/seagate/cortx/s3/bin/s3_start --service s3bgconsumer -c yaml:///etc/cortx/cluster.conf; + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} From 4836af8600028bfd47fa833a2fdb0cab90746ca1 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 22 Oct 2021 11:20:59 -0600 Subject: [PATCH 05/12] UDX-6042 Integration updates * Clean up destroy script * Added VERSION file * Added PVC and PV checking to the status script --- k8_cortx_cloud/VERSION | 1 + k8_cortx_cloud/destroy-cortx-cloud.sh | 2 - k8_cortx_cloud/status-cortx-cloud.sh | 328 +++++++++++++++++++++++++- 3 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 k8_cortx_cloud/VERSION diff --git a/k8_cortx_cloud/VERSION b/k8_cortx_cloud/VERSION new file mode 100644 index 00000000..09c70560 --- /dev/null +++ b/k8_cortx_cloud/VERSION @@ -0,0 +1 @@ +VERSION v0.0.8 \ No newline at end of file diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index d6d9157a..d90a0751 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -109,8 +109,6 @@ printf "########################################################\n" printf "# Delete CORTX GlusterFS \n" printf "########################################################\n" gluster_vol="myvol" -gluster_folder="/etc/gluster" -pod_ctr_mount_path="/mnt/fs-local-volume/$gluster_folder" # Build Gluster endpoint array gluster_ep_array=[] diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh index 20fa8450..5780e80a 100755 --- a/k8_cortx_cloud/status-cortx-cloud.sh +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -142,11 +142,85 @@ else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" fi +# Check storage local +count=0 +num_pvs_pvcs=2 +printf "${INFO}| Checking Storage: Local [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'cortx-control-fs-local-pvc')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'cortx-control-fs-local-pvc')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check storage shared +count=0 +num_pvs_pvcs=2 +printf "${INFO}| Checking Storage: Shared [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'gluster-claim')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'gluster-claim')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + ######################################################################################### # CORTX Data ######################################################################################### nodes_names=$(parseSolution 'solution.nodes.node*.name') num_nodes=$(echo $nodes_names | grep -o '>' | wc -l) +device_names=$(parseSolution 'solution.storage.cvg*.devices*.device') +num_devices=$(echo $device_names | grep -o '>' | wc -l) printf "${ALERT}######################################################${NC}\n" printf "${ALERT}# CORTX Data ${NC}\n" @@ -263,6 +337,114 @@ else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" fi +# Check storage local +count=0 +num_pvs_pvcs=$(($num_nodes*2)) +printf "${INFO}| Checking Storage: Local [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'cortx-data-fs-local-pvc')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'cortx-data-fs-local-pvc')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check storage shared +count=0 +num_pvs_pvcs=2 +printf "${INFO}| Checking Storage: Shared [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'gluster-claim')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'gluster-claim')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + +# Check storage block devices +count=0 +num_pvs_pvcs=$((($num_nodes*$num_devices)*2)) +printf "${INFO}| Checking Storage: Block Devices [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'cortx-data-' | grep -v 'cortx-data-fs-local-pvc')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'cortx-data-' | grep -v 'cortx-data-fs-local-pvc')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + ######################################################################################### # 3rd Party ######################################################################################### @@ -358,6 +540,42 @@ else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" fi +# Check storage local +count=0 +num_pvs_pvcs=$((($num_replicas*2)*3)) +printf "${INFO}| Checking Storage: Local [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep '3rd-party-openldap')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep '3rd-party-openldap')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + printf "${ALERT}### Kafka${NC}\n" # Check StatefulSet num_items=1 @@ -453,6 +671,42 @@ else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" fi +# Check storage local +count=0 +num_pvs_pvcs=$(($num_replicas*2)) +printf "${INFO}| Checking Storage: Local [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'kafka-')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'kafka-')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + printf "${ALERT}### Zookeeper${NC}\n" # Check StatefulSet num_items=1 @@ -548,6 +802,42 @@ else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" fi +# Check storage local +count=0 +num_pvs_pvcs=$(($num_replicas*2)) +printf "${INFO}| Checking Storage: Local [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'zookeeper-')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'zookeeper-')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi + printf "${ALERT}### Consul${NC}\n" # Check StatefulSet num_items=1 @@ -641,4 +931,40 @@ if [[ $num_items -eq $count ]]; then printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" else printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" -fi \ No newline at end of file +fi + +# Check storage local +count=0 +num_pvs_pvcs=$(($num_replicas*2)) +printf "${INFO}| Checking Storage: Local [PVCs/PVs] |${NC}\n" +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PVC: ${status[0]}..." + if [[ "${status[1]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pvc --namespace=$namespace | grep 'consul-server-')" + +while IFS= read -r line; do + IFS=" " read -r -a status <<< "$line" + if [[ "${status[0]}" != "" ]]; then + printf "PV: ${status[5]}..." + if [[ "${status[4]}" != "Bound" ]]; then + printf "${FAILED}FAILED${NC}\n" + else + printf "${PASSED}PASSED${NC}\n" + count=$((count+1)) + fi + fi +done <<< "$(kubectl get pv --namespace=$namespace | grep 'consul-server-')" + +if [[ $num_pvs_pvcs -eq $count ]]; then + printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" +else + printf "OVERALL STATUS: ${FAILED}FAILED${NC}\n" +fi From fe9a4d246639a8cfc635bfa9c895031f51d707e1 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 22 Oct 2021 11:45:39 -0600 Subject: [PATCH 06/12] Updated README.txt --- k8_cortx_cloud/README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8_cortx_cloud/README.txt b/k8_cortx_cloud/README.txt index c09852b3..acdca653 100644 --- a/k8_cortx_cloud/README.txt +++ b/k8_cortx_cloud/README.txt @@ -97,6 +97,6 @@ solution: openldap: ghcr.io/seagate/symas-openldap:standalone consul: hashicorp/consul:1.10.0 kafka: bitnami/kafka:3.0.0-debian-10-r7 - zookeeper: zookeeper:3.7.0-debian-10-r182 + zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 gluster: docker.io/gluster/gluster-centos:latest rancher: rancher/local-path-provisioner:v0.0.20 \ No newline at end of file From aaf7b94e835a601e59363d28be96d448385d856f Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Fri, 22 Oct 2021 12:21:23 -0600 Subject: [PATCH 07/12] UDX-6308_switch_over_from_dummy_to_CORTX_containers - implement the switch from dummy container to cortx container for all containers --- k8_cortx_cloud/README.txt | 13 ++++++++----- .../cortx-control/templates/cortx-control-pod.yaml | 4 ++++ .../cortx-data/templates/cortx-data-pod.yaml | 4 ++++ k8_cortx_cloud/deploy-cortx-cloud.sh | 5 ++++- k8_cortx_cloud/destroy-cortx-cloud.sh | 2 +- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/k8_cortx_cloud/README.txt b/k8_cortx_cloud/README.txt index c09852b3..29a1b310 100644 --- a/k8_cortx_cloud/README.txt +++ b/k8_cortx_cloud/README.txt @@ -50,18 +50,21 @@ Shared glusterFS folder on the worker nodes and inside the Pod containers is loc ########################################################### # Replacing a dummy container with real CORTX container # ########################################################### -See the following example from CORTX Data helm chart, replace the image and -command section hightlighted with "<<===" with the relevant CORTX container -commands required for the entrypoint. An "args" section also can -be added to provide additional arguments. +See the following example from CORTX Data helm chart, replace the command section +hightlighted with "<<===" with the relevant CORTX container commands required for +the entrypoint. An "args" section also can be added to provide additional arguments. ./k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml containers: -- name: cortx-s3-server +- name: cortx-s3-haproxy image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq $.Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: ["/bin/sleep", "3650d"] <<=== + {{- end }} volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml index efa2ec88..4561162a 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-control/templates/cortx-control-pod.yaml @@ -41,7 +41,11 @@ spec: - name: cortx-free-space-monitor image: {{ .Values.cortxcontrol.image }} imagePullPolicy: IfNotPresent + {{- if eq .Values.cortxcontrol.image "centos:7" }} command: ["/bin/sleep", "3650d"] + {{- else }} + command: ["/bin/sleep", "3650d"] + {{- end }} volumeMounts: - name: {{ .Values.cortxcontrol.cfgmap.volmountname }} mountPath: {{ .Values.cortxcontrol.cfgmap.mountpath }} diff --git a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml index f8a91064..64aa30bd 100644 --- a/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml +++ b/k8_cortx_cloud/cortx-cloud-helm-pkg/cortx-data/templates/cortx-data-pod.yaml @@ -201,7 +201,11 @@ spec: - name: {{ printf "cortx-motr-client-%03d" (add 1 $i) }} image: {{ $.Values.cortxdata.image }} imagePullPolicy: IfNotPresent + {{- if eq $.Values.cortxdata.image "centos:7" }} + command: ["/bin/sleep", "3650d"] + {{- else }} command: ["/bin/sleep", "3650d"] + {{- end }} volumeDevices: {{- range $.Files.Lines $.Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} diff --git a/k8_cortx_cloud/deploy-cortx-cloud.sh b/k8_cortx_cloud/deploy-cortx-cloud.sh index 31d044bf..1a875ec5 100755 --- a/k8_cortx_cloud/deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/deploy-cortx-cloud.sh @@ -187,7 +187,10 @@ helm repo add hashicorp https://helm.releases.hashicorp.com if [[ $storage_class == "local-path" ]] then printf "Install Rancher Local Path Provisioner" - rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/rancher-provisioner" + rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" + # Clean up auto gen Rancher Provisioner folder in case it still exists and was not + # clearned up previously by the destroy-cortx-cloud script. + rm -rf $rancher_prov_path mkdir -p $rancher_prov_path rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" cp $(pwd)/cortx-cloud-3rd-party-pkg/templates/local-path-storage-template.yaml $rancher_prov_file diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index d6d9157a..7d4db4df 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -260,7 +260,7 @@ printf "# Delete Consul #\n" printf "########################################################\n" helm delete consul -rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/rancher-provisioner" +rancher_prov_path="$(pwd)/cortx-cloud-3rd-party-pkg/auto-gen-rancher-provisioner" rancher_prov_file="$rancher_prov_path/local-path-storage.yaml" kubectl delete -f $rancher_prov_file rm -rf $rancher_prov_path From 2843da1de7006263fcac9661a49124ea558b1168 Mon Sep 17 00:00:00 2001 From: danthanhton <87337002+danthanhton@users.noreply.github.com> Date: Fri, 22 Oct 2021 12:41:31 -0600 Subject: [PATCH 08/12] UDX-6308_switch_over_from_dummy_to_CORTX_containers - Add /etc/gluster/var/log/cortx back after cleaning up everything in /etc/gluster --- k8_cortx_cloud/destroy-cortx-cloud.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8_cortx_cloud/destroy-cortx-cloud.sh b/k8_cortx_cloud/destroy-cortx-cloud.sh index 7d4db4df..7f612f4f 100755 --- a/k8_cortx_cloud/destroy-cortx-cloud.sh +++ b/k8_cortx_cloud/destroy-cortx-cloud.sh @@ -137,7 +137,9 @@ do printf "Stop and delete GlusterFS volume: $gluster_node_name \n" printf "=================================================================================\n" kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ - 'rm -rf /etc/gluster/* /etc/gluster/.glusterfs/' + 'rm -rf /etc/gluster/* /etc/gluster/.glusterfs/' + kubectl exec --namespace=$namespace -i $gluster_node_name -- bash -c \ + 'mkdir -p /etc/gluster/var/log/cortx' if [[ "$count" == 0 ]]; then first_gluster_node_name=$gluster_node_name echo y | kubectl exec --namespace=$namespace -i $gluster_node_name -- gluster volume stop $gluster_vol From f893543132ae51fa67976458fe2ee1ae468f4cd5 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 22 Oct 2021 12:58:52 -0600 Subject: [PATCH 09/12] UDX-6042 Integration updates * Added solution_dummy.yaml * Change order of mk.fs in preq script * Updated status to use the correct namespace for 3rd party --- k8_cortx_cloud/VERSION | 2 +- k8_cortx_cloud/prereq-deploy-cortx-cloud.sh | 16 +++-- k8_cortx_cloud/solution_dummy.yaml | 73 +++++++++++++++++++++ k8_cortx_cloud/status-cortx-cloud.sh | 16 ++--- 4 files changed, 91 insertions(+), 16 deletions(-) create mode 100644 k8_cortx_cloud/solution_dummy.yaml diff --git a/k8_cortx_cloud/VERSION b/k8_cortx_cloud/VERSION index 09c70560..c7c489e2 100644 --- a/k8_cortx_cloud/VERSION +++ b/k8_cortx_cloud/VERSION @@ -1 +1 @@ -VERSION v0.0.8 \ No newline at end of file +VERSION v0.0.9 \ No newline at end of file diff --git a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh index d1c0bf70..12e2ba0e 100755 --- a/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh +++ b/k8_cortx_cloud/prereq-deploy-cortx-cloud.sh @@ -113,6 +113,15 @@ sysctl -w vm.max_map_count=30000000; printf "####################################################\n" printf "# Prep for CORTX deployment \n" printf "####################################################\n" + +if [[ $(findmnt -m $fs_mount_path) ]];then + echo "$fs_mount_path already mounted..." +else + mkdir -p $fs_mount_path + echo y | mkfs.ext4 $disk + mount -t ext4 $disk $fs_mount_path +fi + # Prep for Rancher Local Path Provisioner deployment echo "Create folder '$fs_mount_path/local-path-provisioner'" mkdir -p $fs_mount_path/local-path-provisioner @@ -128,13 +137,6 @@ while true; do sleep 1s done -if [[ $(findmnt -m $fs_mount_path) ]];then - echo "$fs_mount_path already mounted..." -else - echo y | mkfs.ext4 $disk - mount -t ext4 $disk $fs_mount_path -fi - # Prep for GlusterFS deployment yum install glusterfs-fuse -y mkdir -p $fs_mount_path/etc/gluster diff --git a/k8_cortx_cloud/solution_dummy.yaml b/k8_cortx_cloud/solution_dummy.yaml new file mode 100644 index 00000000..07b7aa7c --- /dev/null +++ b/k8_cortx_cloud/solution_dummy.yaml @@ -0,0 +1,73 @@ +solution: + namespace: default + secrets: + name: cortx-secret + content: + openldap_admin_secret: seagate1 + kafka_admin_secret: Seagate@123 + consul_admin_secret: Seagate@123 + common_admin_secret: Seagate@123 + s3_auth_admin_secret: ldapadmin + csm_auth_admin_secret: seagate2 + csm_mgmt_admin_secret: Cortxadmin@123 + images: + cortxcontrolprov: centos:7 + cortxcontrol: centos:7 + cortxdataprov: centos:7 + cortxdata: centos:7 + openldap: ghcr.io/seagate/symas-openldap:standalone + consul: hashicorp/consul:1.10.0 + kafka: bitnami/kafka:3.0.0-debian-10-r7 + zookeeper: bitnami/zookeeper:3.7.0-debian-10-r182 + gluster: docker.io/gluster/gluster-centos:latest + rancher: rancher/local-path-provisioner:v0.0.20 + common: + cortx_io_svc_ingress: false + storage_provisioner_path: /mnt/fs-local-volume + storage: + local: /etc/cortx + shared: /share + log: /share/var/log/cortx + s3: + num_inst: 2 + start_port_num: 28051 + motr: + num_client_inst: 0 + start_port_num: 29000 + storage_sets: + name: storage-set-1 + durability: + sns: 1+0+0 + dix: 1+0+0 + storage: + cvg1: + name: cvg-01 + type: ios + devices: + metadata: + device: /dev/sdc + size: 5Gi + data: + d1: + device: /dev/sdd + size: 5Gi + cvg2: + name: cvg-02 + type: ios + devices: + metadata: + device: /dev/sde + size: 5Gi + data: + d1: + device: /dev/sdf + size: 5Gi + nodes: + node1: + name: node-1 + node2: + name: node-2 + node3: + name: node-3 + node4: + name: node-4 diff --git a/k8_cortx_cloud/status-cortx-cloud.sh b/k8_cortx_cloud/status-cortx-cloud.sh index 5780e80a..ea1d510d 100755 --- a/k8_cortx_cloud/status-cortx-cloud.sh +++ b/k8_cortx_cloud/status-cortx-cloud.sh @@ -555,7 +555,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pvc --namespace=$namespace | grep '3rd-party-openldap')" +done <<< "$(kubectl get pvc | grep '3rd-party-openldap')" while IFS= read -r line; do IFS=" " read -r -a status <<< "$line" @@ -568,7 +568,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pv --namespace=$namespace | grep '3rd-party-openldap')" +done <<< "$(kubectl get pv | grep '3rd-party-openldap')" if [[ $num_pvs_pvcs -eq $count ]]; then printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" @@ -686,7 +686,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pvc --namespace=$namespace | grep 'kafka-')" +done <<< "$(kubectl get pvc | grep 'kafka-')" while IFS= read -r line; do IFS=" " read -r -a status <<< "$line" @@ -699,7 +699,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pv --namespace=$namespace | grep 'kafka-')" +done <<< "$(kubectl get pv | grep 'kafka-')" if [[ $num_pvs_pvcs -eq $count ]]; then printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" @@ -817,7 +817,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pvc --namespace=$namespace | grep 'zookeeper-')" +done <<< "$(kubectl get pvc | grep 'zookeeper-')" while IFS= read -r line; do IFS=" " read -r -a status <<< "$line" @@ -830,7 +830,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pv --namespace=$namespace | grep 'zookeeper-')" +done <<< "$(kubectl get pv | grep 'zookeeper-')" if [[ $num_pvs_pvcs -eq $count ]]; then printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" @@ -948,7 +948,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pvc --namespace=$namespace | grep 'consul-server-')" +done <<< "$(kubectl get pvc | grep 'consul-server-')" while IFS= read -r line; do IFS=" " read -r -a status <<< "$line" @@ -961,7 +961,7 @@ while IFS= read -r line; do count=$((count+1)) fi fi -done <<< "$(kubectl get pv --namespace=$namespace | grep 'consul-server-')" +done <<< "$(kubectl get pv | grep 'consul-server-')" if [[ $num_pvs_pvcs -eq $count ]]; then printf "OVERALL STATUS: ${PASSED}PASSED${NC}\n" From 312dba432ca5f595acbbd2b03476cd0d20cd96b4 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 22 Oct 2021 13:00:07 -0600 Subject: [PATCH 10/12] Updated version back to v0.0.8 --- k8_cortx_cloud/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8_cortx_cloud/VERSION b/k8_cortx_cloud/VERSION index c7c489e2..09c70560 100644 --- a/k8_cortx_cloud/VERSION +++ b/k8_cortx_cloud/VERSION @@ -1 +1 @@ -VERSION v0.0.9 \ No newline at end of file +VERSION v0.0.8 \ No newline at end of file From e26f120e58431fdc5e49effd66f3c1b8c00e4218 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 22 Oct 2021 14:20:54 -0600 Subject: [PATCH 11/12] Updates to README.txt --- k8_cortx_cloud/README.txt | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/k8_cortx_cloud/README.txt b/k8_cortx_cloud/README.txt index 0f11550c..d86c503e 100644 --- a/k8_cortx_cloud/README.txt +++ b/k8_cortx_cloud/README.txt @@ -50,6 +50,18 @@ Shared glusterFS folder on the worker nodes and inside the Pod containers is loc ########################################################### # Replacing a dummy container with real CORTX container # ########################################################### + +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +The Helm charts work with both "dummy" and "CORTX ALL" containers. +If image is centos:7 helm runs in "dummy" mode any other name runs "CORTX ALL" mode + +{- if eq $.Values.cortxdata.image "centos:7" }} # DO NOT CHANGE +command: ["/bin/sleep", "3650d"] # DO NOT CHANGE +{{- else }} # DO NOT CHANGE +command: ["/bin/sleep", "3650d"] #<<=========================== REPLACE THIS WITH THE CORTX ENTRY POINT +{{- end }} # DO NOT CHANGE +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + See the following example from CORTX Data helm chart, replace the command section hightlighted with "<<===" with the relevant CORTX container commands required for the entrypoint. An "args" section also can be added to provide additional arguments. @@ -60,11 +72,11 @@ containers: - name: cortx-s3-haproxy image: {{ .Values.cortxdata.image }} imagePullPolicy: IfNotPresent - {{- if eq $.Values.cortxdata.image "centos:7" }} - command: ["/bin/sleep", "3650d"] - {{- else }} - command: ["/bin/sleep", "3650d"] <<=== - {{- end }} + {- if eq $.Values.cortxdata.image "centos:7" }} # DO NOT CHANGE + command: ["/bin/sleep", "3650d"] # DO NOT CHANGE + {{- else }} # DO NOT CHANGE + command: ["/bin/sleep", "3650d"] #<<=========================== REPLACE THIS WITH THE CORTX ENTRY POINT + {{- end }} # DO NOT CHANGE volumeDevices: {{- range .Files.Lines .Values.cortxdata.mountblkinfo }} - name: {{ printf "cortx-data-%s-pv-%s" ( base .) $nodename }} @@ -93,10 +105,10 @@ The images can be changed by modifying the solution.yaml file section solution.i solution: namespace: default images: - cortxcontrolprov: centos:7 - cortxcontrol: centos:7 - cortxdataprov: centos:7 - cortxdata: centos:7 + cortxcontrolprov: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxcontrol: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxdataprov: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci + cortxdata: ghcr.io/seagate/cortx-all:2.0.0-latest-custom-ci openldap: ghcr.io/seagate/symas-openldap:standalone consul: hashicorp/consul:1.10.0 kafka: bitnami/kafka:3.0.0-debian-10-r7 From aa46db20ca29e090dd3553806d4d01dc99420db1 Mon Sep 17 00:00:00 2001 From: John Fletcher Date: Fri, 22 Oct 2021 14:21:34 -0600 Subject: [PATCH 12/12] Updated version to v0.0.9 --- k8_cortx_cloud/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8_cortx_cloud/VERSION b/k8_cortx_cloud/VERSION index 09c70560..c7c489e2 100644 --- a/k8_cortx_cloud/VERSION +++ b/k8_cortx_cloud/VERSION @@ -1 +1 @@ -VERSION v0.0.8 \ No newline at end of file +VERSION v0.0.9 \ No newline at end of file