K8SPSMDB-1212: Don't disable balancer during restore

e2e-tests/balancer/conf/backup-minio.yml

@@ -0,0 +1,9 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBBackup
+metadata:
+  finalizers:
+    - percona.com/delete-backup
+  name: backup-minio
+spec:
+  clusterName: some-name
+  storageName: minio

e2e-tests/balancer/conf/restore.yml

@@ -0,0 +1,7 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBRestore
+metadata:
+  name:
+spec:
+  clusterName: some-name
+  backupName:

e2e-tests/balancer/conf/some-name-rs0.yml

@@ -7,7 +7,17 @@ spec:
   image:
   imagePullPolicy: Always
   backup:
-    enabled: false
+    enabled: true
+    image: perconalab/percona-server-mongodb-operator:1.1.0-backup
+    storages:
+      minio:
+        type: s3
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing
+          endpointUrl: http://minio-service:9000/
+          insecureSkipTLSVerify: false
   sharding:
     enabled: true
     configsvrReplSet:

e2e-tests/balancer/run

@@ -1,63 +1,77 @@
 #!/bin/bash
 
 set -o errexit
-set -o xtrace
 
 test_dir=$(realpath "$(dirname "$0")")
 . "${test_dir}/../functions"
 set_debug
 
-check_balancer() {
-	local expected=$1 # should be "full" (running balancer) or "off" (disabled balancer)
+log() {
+	echo "[$(date +%Y-%m-%dT%H:%M:%S%z)]" $*
+}
 
+check_balancer() {
+	local cluster=$1
+	local expected=$2 # should be "true" (enabled) or "false" (disabled)
+	local delay=${3:-"0"}
 	local balancer_running
-	balancer_running=$(run_mongos 'db.adminCommand({balancerStatus: 1}).mode' "clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" \
-		| grep -E -v "Percona Server for MongoDB|connecting to:|Implicit session:|versions do not match|Error saving history file:|bye")
 
+	log "sleeping for ${delay} seconds..."
+	sleep ${delay}
+
+	balancer_running=$(run_mongosh 'sh.getBalancerState()' "clusterAdmin:clusterAdmin123456@${cluster}-cfg.${namespace}" \
+		| grep -E -v 'Warning|cfg' | grep -E 'true|false')
+
+	echo -n "checking if balancer status is ${expected}..."
 	if [[ $balancer_running != "$expected" ]]; then
-		echo "Unexpected output from \"db.adminCommand({balancerStatus: 1}).mode\": $balancer_running"
-		echo "Expected $expected"
+		echo
+		log "Unexpected output from \"sh.getBalancerState()\": $balancer_running"
+		log "Expected: $expected"
 		exit 1
 	fi
+	echo "OK"
 }
 
-check_service() {
-	state=$1
-	svc_name=$2
-	if [ $state = "present" ]; then
-		echo -n "check that $svc_name was created"
-		local timeout=0
-		until kubectl_bin get service/$svc_name -o 'jsonpath={.spec.type}' 2>&1 | grep -vq NotFound; do
-			sleep 1
-			timeout=$((timeout + 1))
-			echo -n '.'
-			if [[ ${timeout} -gt 900 ]]; then
-				echo "Waiting timeout has been reached. Service $svc_name is not present. Exiting..."
-				exit 1
-			fi
-		done
-		echo ".OK"
-	elif [ $state = "removed" ]; then
-		echo -n "check that $svc_name was removed"
-		if [[ -z $(kubectl_bin get service/$svc_name -o 'jsonpath={.spec.type}' 2>&1 | grep NotFound) ]]; then
-			echo "$svc_name was not removed."
-			exit 1
-		else
-			echo ".OK"
-		fi
-	else
-		echo "unknown state $state"
-	fi
+check_backup_and_restore() {
+	local cluster=$1
+	local backup_suffix=$2
+	local balancer_end_state=$3
+	local backup_name="backup-minio-${backup_suffix}"
+
+	log "running backup: ${backup_name}"
+	run_backup "minio" "${backup_name}"
+	wait_backup "${backup_name}" "requested"
+
+	log "checking if balancer is disabled"
+	check_balancer ${cluster} "false"
+
+	wait_backup "${backup_name}" "ready"
+
+	log "checking if balancer is ${balancer_end_state} after backup"
+	check_balancer ${cluster} ${balancer_end_state} 10
+
+	log "running restore: restore-${backup_name}"
+	run_restore "${backup_name}"
+	log "checking if balancer status is not changed"
+	check_balancer ${cluster} "${balancer_end_state}" 4
+
+	wait_restore ${backup_name} ${cluster} "ready"
+
+	log "checking if balancer is ${balancer_end_state} after restore"
+	check_balancer ${cluster} ${balancer_end_state} 10
 }
 
 main() {
 	create_infra "$namespace"
 
+	deploy_minio
+	apply_s3_storage_secrets
+
 	desc 'create first PSMDB cluster'
 	cluster="some-name"
 	kubectl_bin apply \
 		-f "$conf_dir/secrets.yml" \
-		-f "$conf_dir/client.yml"
+		-f "$conf_dir/client-70.yml"
 
 	if version_gt "1.19" && [ $EKS -ne 1 ]; then
 		$sed 's/docker/runc/g' "$conf_dir/container-rc.yaml" | kubectl_bin apply -f -
@@ -70,30 +84,22 @@ main() {
 	apply_cluster "$test_dir/conf/$cluster-rs0.yml"
 
 	desc 'check if all 3 Pods started'
-	wait_for_running $cluster-rs0 3
+	wait_for_running $cluster-rs0 3 "false"
 	wait_for_running $cluster-cfg 3 "false"
 	wait_for_running $cluster-mongos 3
-	sleep 20
-	check_balancer "full"
+	check_balancer ${cluster} "true" 10
+
+	check_backup_and_restore ${cluster} "0" "true"
 
 	desc 'disabling balancer'
 	kubectl patch psmdb some-name --type=merge -p '{"spec":{"sharding":{"balancer":{"enabled":false}}}}'
-	sleep 20
-	check_balancer "off"
+	check_balancer ${cluster} "false" 10
+
+	check_backup_and_restore ${cluster} "1" "false"
 
 	desc 'enabling balancer'
 	kubectl patch psmdb some-name --type=merge -p '{"spec":{"sharding":{"balancer":{"enabled":true}}}}'
-	sleep 20
-	check_balancer "full"
-
-	# Add check that servicePerPod creates 3 services for the running cluster
-	desc 'enabling servicePerPod for mongos'
-	kubectl patch psmdb some-name --type=merge -p '{"spec":{"sharding":{"mongos":{"expose":{"servicePerPod":true}}}}}'
-	wait_for_running $cluster-mongos 3
-	check_service present $cluster-mongos-0
-	check_service present $cluster-mongos-1
-	check_service present $cluster-mongos-2
-	check_service removed $cluster-mongos
+	check_balancer ${cluster} "true" 10
 
 	destroy "$namespace"
 }

e2e-tests/conf/client-70.yml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: psmdb-client
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: psmdb-client
+  template:
+    metadata:
+      labels:
+        name: psmdb-client
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+        - name: psmdb-client
+          image: percona/percona-server-mongodb:7.0
+          imagePullPolicy: Always
+          command:
+          - sleep
+          args:
+          - "100500"

e2e-tests/functions

@@ -218,12 +218,13 @@ wait_backup_agent() {
 
 wait_backup() {
 	local backup_name=$1
+	local target_state=${2:-"ready"}
 
 	set +o xtrace
 	retry=0
-	echo -n $backup_name
+	echo -n "waiting for ${backup_name} to reach ${target_state} state"
 	local current_status=
-	until [[ ${current_status} == "ready" ]]; do
+	until [[ ${current_status} == ${target_state} ]]; do
 		sleep 1
 		echo -n .
 		let retry+=1
@@ -344,7 +345,7 @@ wait_restore() {
 
 	set +o xtrace
 	retry=0
-	echo -n "waiting psmdb-restore/${backup_name} to reach ${target_state} state"
+	echo -n "waiting psmdb-restore/restore-${backup_name} to reach ${target_state} state"
 	local current_state=
 	until [[ ${current_state} == ${target_state} ]]; do
 		sleep 1
@@ -733,9 +734,22 @@ run_mongo() {
 	[[ $uri == *cfg* ]] && replica_set='cfg' || replica_set='rs0'
 	kubectl_bin exec ${client_container} -- \
 		bash -c "printf '$command\n' | mongo $driver://$uri$suffix/admin?ssl=false\&replicaSet=$replica_set $mongo_flag"
+}
+
+run_mongosh() {
+	local command="$1"
+	local uri="$2"
+	local driver=${3:-"mongodb+srv"}
+	local suffix=${4:-".svc.cluster.local"}
+	local mongo_flag="$5"
 
+	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
+	[[ $uri == *cfg* ]] && replica_set='cfg' || replica_set='rs0'
+	kubectl_bin exec ${client_container} -- \
+		bash -c "printf '$command\n' | mongosh --quiet $driver://$uri$suffix/admin?ssl=false\&replicaSet=$replica_set $mongo_flag"
 }
 
+
 run_mongo_tls() {
 	local command="$1"
 	local uri="$2"

e2e-tests/service-per-pod/conf/sharded.yml

@@ -0,0 +1,51 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: some-name
+spec:
+  #platform: openshift
+  image:
+  imagePullPolicy: Always
+  backup:
+    enabled: false
+  sharding:
+    enabled: true
+    configsvrReplSet:
+      size: 3
+      volumeSpec:
+        persistentVolumeClaim:
+          resources:
+            requests:
+              storage: 3Gi
+      expose:
+        enabled: false
+
+    mongos:
+      size: 3
+      configuration: |
+        replication:
+          localPingThresholdMs: 15
+      expose:
+        type: ClusterIP
+
+  replsets:
+  - name: rs0
+    affinity:
+      antiAffinityTopologyKey: none
+    expose:
+      enabled: false
+    resources:
+      limits:
+        cpu: 500m
+        memory: 1G
+      requests:
+        cpu: 100m
+        memory: 0.1G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    size: 3
+  secrets:
+    users: some-users

e2e-tests/service-per-pod/run

@@ -9,7 +9,7 @@ set_debug
 check_cr_config() {
 	local cluster="$1"
 
-	desc "create first PSMDB cluster $cluster"
+	desc "create PSMDB cluster $cluster"
 	apply_cluster $test_dir/conf/$cluster.yml
 
 	desc 'check if all 3 Pods started'
@@ -40,8 +40,8 @@ check_cr_config() {
 	compare_mongo_cmd "find" "myApp:myPass@$(get_service_ip $cluster-1)" "" ":27017"
 	compare_mongo_cmd "find" "myApp:myPass@$(get_service_ip $cluster-2)" "" ":27017"
 
-	desc 'add service-per-pod lebel and annotation'
 	if [[ $cluster == "node-port-rs0" ]]; then
+		desc 'add service-per-pod label and annotation'
 		old_node_port=$(kubectl_bin get svc node-port-rs0-0 -o 'jsonpath={.spec.ports[0].nodePort}')
 		kubectl_bin patch psmdb node-port --type=json --patch '[
 		{
@@ -74,6 +74,36 @@ check_cr_config() {
 		-f $test_dir/conf/$cluster.yml
 }
 
+check_service() {
+	state=$1
+	svc_name=$2
+	if [ $state = "present" ]; then
+		echo -n "check that $svc_name was created"
+		local timeout=0
+		until kubectl_bin get service/$svc_name -o 'jsonpath={.spec.type}' 2>&1 | grep -vq NotFound; do
+			sleep 1
+			timeout=$((timeout + 1))
+			echo -n '.'
+			if [[ ${timeout} -gt 900 ]]; then
+				echo "Waiting timeout has been reached. Service $svc_name is not present. Exiting..."
+				exit 1
+			fi
+		done
+		echo ".OK"
+	elif [ $state = "removed" ]; then
+		echo -n "check that $svc_name was removed"
+		if [[ -z $(kubectl_bin get service/$svc_name -o 'jsonpath={.spec.type}' 2>&1 | grep NotFound) ]]; then
+			echo "$svc_name was not removed."
+			exit 1
+		else
+			echo ".OK"
+		fi
+	else
+		echo "unknown state $state"
+	fi
+}
+
+
 main() {
 	create_infra $namespace
 	deploy_cert_manager
@@ -92,6 +122,21 @@ main() {
 	desc 'check NodePort'
 	check_cr_config "node-port-rs0"
 
+	desc 'check Mongos in sharded cluster'
+	local cluster=some-name
+	apply_cluster "$test_dir/conf/sharded.yml"
+	wait_for_running $cluster-rs0 3
+	wait_for_running $cluster-cfg 3 "false"
+	wait_for_running $cluster-mongos 3
+
+	desc 'enabling servicePerPod for mongos'
+	kubectl patch psmdb some-name --type=merge -p '{"spec":{"sharding":{"mongos":{"expose":{"servicePerPod":true}}}}}'
+	wait_for_running $cluster-mongos 3
+	check_service present $cluster-mongos-0
+	check_service present $cluster-mongos-1
+	check_service present $cluster-mongos-2
+	check_service removed $cluster-mongos
+
 	destroy $namespace
 
 	desc 'test passed'