Merge pull request #1781 from percona/K8SPSMDB-1249

K8SPSMDB-1249: Fix smart update for pods that are not member of replset
percona · Jan 14, 2025 · afc19ed · afc19ed
2 parents 86826ed + 901284e
commit afc19ed
Show file tree

Hide file tree

Showing 18 changed files with 256 additions and 117 deletions.
diff --git a/Makefile b/Makefile
@@ -60,6 +60,7 @@ uninstall: manifests ## Uninstall CRDs, rbac
 deploy: ## Deploy operator
 	yq eval '(.spec.template.spec.containers[] | select(.name=="percona-server-mongodb-operator")).image = "$(IMAGE)"' $(DEPLOYDIR)/operator.yaml \
 		| yq eval '(.spec.template.spec.containers[] | select(.name=="percona-server-mongodb-operator").env[] | select(.name=="LOG_LEVEL")).value="DEBUG"' - \
+		| yq eval '(.spec.template.spec.containers[] | select(.name=="percona-server-mongodb-operator").env[] | select(.name=="DISABLE_TELEMETRY")).value="true"' - \
 		| kubectl apply -f -
 
 undeploy: ## Undeploy operator

diff --git a/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml b/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml
@@ -18849,14 +18849,16 @@ spec:
                     initialized:
                       type: boolean
                     members:
-                      items:
+                      additionalProperties:
                         properties:
                           name:
                             type: string
-                          version:
+                          state:
+                            type: integer
+                          stateStr:
                             type: string
                         type: object
-                      type: array
+                      type: object
                     message:
                       type: string
                     ready:

diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml
@@ -19545,14 +19545,16 @@ spec:
                     initialized:
                       type: boolean
                     members:
-                      items:
+                      additionalProperties:
                         properties:
                           name:
                             type: string
-                          version:
+                          state:
+                            type: integer
+                          stateStr:
                             type: string
                         type: object
-                      type: array
+                      type: object
                     message:
                       type: string
                     ready:

diff --git a/deploy/crd.yaml b/deploy/crd.yaml
@@ -19545,14 +19545,16 @@ spec:
                     initialized:
                       type: boolean
                     members:
-                      items:
+                      additionalProperties:
                         properties:
                           name:
                             type: string
-                          version:
+                          state:
+                            type: integer
+                          stateStr:
                             type: string
                         type: object
-                      type: array
+                      type: object
                     message:
                       type: string
                     ready:

diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml
@@ -19545,14 +19545,16 @@ spec:
                     initialized:
                       type: boolean
                     members:
-                      items:
+                      additionalProperties:
                         properties:
                           name:
                             type: string
-                          version:
+                          state:
+                            type: integer
+                          stateStr:
                             type: string
                         type: object
-                      type: array
+                      type: object
                     message:
                       type: string
                     ready:

diff --git a/e2e-tests/default-cr/run b/e2e-tests/default-cr/run
@@ -43,24 +43,34 @@ function start_cluster() {
 }
 
 function main() {
-	create_infra "$namespace"
-	cluster="my-cluster-name"
-
-	desc 'create secrets and start client'
-	kubectl_bin apply -f $deploy_dir/secrets.yaml
-	kubectl_bin apply -f $conf_dir/client.yml
-
-	desc "create first PSMDB cluster $cluster"
+	delete_crd
+	check_crd_for_deletion "${GIT_BRANCH}"
 	kubectl_bin apply ${OPERATOR_NS:+-n $OPERATOR_NS} --server-side --force-conflicts -f $deploy_dir/crd.yaml
+
 	if [ -n "$OPERATOR_NS" ]; then
+		create_namespace $OPERATOR_NS
+		create_namespace ${namespace}
 		apply_rbac cw-rbac
-		kubectl_bin apply -n ${OPERATOR_NS} -f $deploy_dir/cw-operator.yaml
+		yq eval '
+			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
+			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' ${src_dir}/deploy/cw-operator.yaml \
+			| kubectl_bin apply -f -
 	else
+		create_namespace ${namespace}
 		apply_rbac rbac
-		yq eval '((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true")' "$deploy_dir/operator.yaml" \
+		yq eval '
+			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
+			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' ${src_dir}/deploy/operator.yaml \
 			| kubectl_bin apply -f -
 	fi
 
+	cluster="my-cluster-name"
+
+	desc 'create secrets and start client'
+	kubectl_bin apply -f $deploy_dir/secrets.yaml
+	kubectl_bin apply -f $conf_dir/client.yml
+
+	desc "create first PSMDB cluster $cluster"
 	yq eval '.spec.upgradeOptions.versionServiceEndpoint = "https://check-dev.percona.com" |
 		.spec.replsets[].affinity.antiAffinityTopologyKey = "none" |
 		.spec.replsets[].nonvoting.affinity.antiAffinityTopologyKey = "none" |

diff --git a/e2e-tests/preinit-updates/conf/some-name.yml b/e2e-tests/preinit-updates/conf/some-name.yml
@@ -0,0 +1,55 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: some-name
+spec:
+  #platform: openshift
+  image:
+  imagePullPolicy: Always
+  allowUnsafeConfigurations: false
+  updateStrategy: SmartUpdate
+  backup:
+    enabled: false
+    image: perconalab/percona-server-mongodb-operator:0.4.0-backup
+  replsets:
+  - name: rs0
+    # readinessDelaySec: 40
+    # livenessDelaySec: 120
+    affinity:
+      antiAffinityTopologyKey: none
+    resources:
+      limits:
+        cpu: 500m
+        memory: 0.5G
+      requests:
+        cpu: 100m
+        memory: 0.1G
+    configuration: |
+      operationProfiling:
+        mode: slowOp
+        slowOpThresholdMs: 100
+      security:
+        enableEncryption: true
+        redactClientLogData: false
+      setParameter:
+        ttlMonitorSleepSecs: 60
+        wiredTigerConcurrentReadTransactions: 128
+        wiredTigerConcurrentWriteTransactions: 128
+      storage:
+        engine: wiredTiger
+        wiredTiger:
+          collectionConfig:
+            blockCompressor: snappy
+          engineConfig:
+            directoryForIndexes: false
+            journalCompressor: snappy
+          indexConfig:
+            prefixCompression: true
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    size: 3
+  secrets:
+    users: some-users
diff --git a/e2e-tests/preinit-updates/run b/e2e-tests/preinit-updates/run
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -o errexit
+
+test_dir=$(realpath $(dirname $0))
+. ${test_dir}/../functions
+set_debug
+
+create_infra ${namespace}
+
+cluster="some-name"
+desc "test starts on cluster: ${cluster}"
+
+apply_cluster ${test_dir}/conf/${cluster}.yml
+wait_for_running ${cluster}-rs0 1 "false"
+echo "enabling backups"
+kubectl_bin patch psmdb ${cluster} --type=merge -p '{"spec":{"backup":{"enabled":true}}}'
+echo "sleeping for 7 seconds..."
+sleep 7
+echo "changing rs0 resources"
+kubectl_bin patch psmdb ${cluster} --type=json -p '[{"op":"replace","path":"/spec/replsets/0/resources/limits/cpu","value":"550m"}]'
+
+wait_for_running ${cluster}-rs0 3
+wait_cluster_consistency "${cluster}"
+
+desc 'test passed'
+
+destroy $namespace
diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv
@@ -31,6 +31,7 @@ operator-self-healing-chaos
 pitr
 pitr-sharded
 pitr-physical
+preinit-updates
 pvc-resize
 recover-no-primary
 replset-overrides

diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv
@@ -32,6 +32,7 @@ operator-self-healing-chaos
 pitr
 pitr-sharded
 pitr-physical
+preinit-updates
 pvc-resize
 recover-no-primary
 replset-overrides

diff --git a/e2e-tests/version-service/conf/crd.yaml b/e2e-tests/version-service/conf/crd.yaml
@@ -19545,14 +19545,16 @@ spec:
                     initialized:
                       type: boolean
                     members:
-                      items:
+                      additionalProperties:
                         properties:
                           name:
                             type: string
-                          version:
+                          state:
+                            type: integer
+                          stateStr:
                             type: string
                         type: object
-                      type: array
+                      type: object
                     message:
                       type: string
                     ready:

diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go
@@ -28,6 +28,7 @@ import (
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
 
 	"github.com/percona/percona-server-mongodb-operator/pkg/mcs"
+	"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo"
 	"github.com/percona/percona-server-mongodb-operator/pkg/util/numstr"
 	"github.com/percona/percona-server-mongodb-operator/version"
 )
@@ -234,8 +235,9 @@ type UpgradeOptions struct {
 }
 
 type ReplsetMemberStatus struct {
-	Name    string `json:"name,omitempty"`
-	Version string `json:"version,omitempty"`
+	Name     string            `json:"name,omitempty"`
+	State    mongo.MemberState `json:"state,omitempty"`
+	StateStr string            `json:"stateStr,omitempty"`
 }
 
 type MongosStatus struct {
@@ -246,8 +248,8 @@ type MongosStatus struct {
 }
 
 type ReplsetStatus struct {
-	Members     []*ReplsetMemberStatus `json:"members,omitempty"`
-	ClusterRole ClusterRole            `json:"clusterRole,omitempty"`
+	Members     map[string]ReplsetMemberStatus `json:"members,omitempty"`
+	ClusterRole ClusterRole                    `json:"clusterRole,omitempty"`
 
 	Initialized  bool     `json:"initialized,omitempty"`
 	AddedAsShard *bool    `json:"added_as_shard,omitempty"`

diff --git a/pkg/apis/psmdb/v1/zz_generated.deepcopy.go b/pkg/apis/psmdb/v1/zz_generated.deepcopy.go
diff --git a/pkg/controller/perconaservermongodb/finalizers.go b/pkg/controller/perconaservermongodb/finalizers.go
@@ -120,7 +120,7 @@ func (r *ReconcilePerconaServerMongoDB) deletePSMDBPods(ctx context.Context, cr
 			rsDeleted = false
 			switch err {
 			case errWaitingTermination, errWaitingFirstPrimary:
-				log.Error(err, "rs", rs.Name)
+				log.Info("deleting rs pods", "rs", rs.Name, "status", err.Error())
 				continue
 			default:
 				log.Error(err, "failed to delete rs pods", "rs", rs.Name)