From 74a345606ea80cbff27f46cd6b582c62100a5fc9 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Wed, 18 Dec 2024 11:13:33 +0100 Subject: [PATCH] Create database schema for Watcher database This commit is implementing db schema creation (dbsync) via Job following the same pattern as other operators. It introduces some spec and status changes required and initial configuration with only the required configuration required to run the dbsync (just the database connection and transport which will be required later to pass it to the subCRs). This patch is also adding a new functional scenario test to test the Watcher controller when using non-default values. Finally, this patch is also skipping two new error types in the crd-schema-checker as it complains about perserveJobs for being a boolean type and Hash to be a map. However we want to keep them with those types for consistency with the rest of operators. --- .../watcher.openstack.org_watcherapis.yaml | 5 + api/bases/watcher.openstack.org_watchers.yaml | 10 ++ api/v1beta1/common_types.go | 5 + api/v1beta1/watcher_types.go | 9 +- api/v1beta1/zz_generated.deepcopy.go | 7 + .../watcher.openstack.org_watcherapis.yaml | 5 + .../bases/watcher.openstack.org_watchers.yaml | 10 ++ config/rbac/role.yaml | 12 ++ controllers/watcher_controller.go | 140 ++++++++++++++++ hack/crd-schema-checker.sh | 2 +- pkg/watcher/constants.go | 9 ++ pkg/watcher/dbsync.go | 108 +++++++++++++ pkg/watcher/volumes.go | 102 ++++++++++++ templates/watcher/config/00-default.conf | 11 ++ .../watcher/config/watcher-dbsync-config.json | 3 + tests/functional/base_test.go | 10 ++ tests/functional/watcher_controller_test.go | 150 ++++++++++++++++++ tests/functional/watcher_test_data.go | 5 + 18 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 pkg/watcher/dbsync.go create mode 100644 pkg/watcher/volumes.go create mode 100644 templates/watcher/config/watcher-dbsync-config.json diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index b27f85a..aa5afc5 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -70,6 +70,11 @@ spec: password from the Secret type: string type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean secret: description: Secret containing all passwords / keys needed type: string diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index 88174e6..ed6213c 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -75,6 +75,11 @@ spec: password from the Secret type: string type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean rabbitMqClusterName: default: rabbitmq description: |- @@ -142,6 +147,11 @@ spec: - type type: object type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object observedGeneration: description: |- ObservedGeneration - the most recent generation observed for this diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 42cb0ae..a862c55 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -52,6 +52,11 @@ type WatcherCommon struct { // +kubebuilder:default=memcached // MemcachedInstance is the name of the Memcached CR that all watcher service will use. MemcachedInstance string `json:"memcachedInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` } // WatcherTemplate defines the fields used in the top level CR diff --git a/api/v1beta1/watcher_types.go b/api/v1beta1/watcher_types.go index 6944ed4..ae36b54 100644 --- a/api/v1beta1/watcher_types.go +++ b/api/v1beta1/watcher_types.go @@ -21,6 +21,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // DbSyncHash hash + DbSyncHash = "dbsync" +) + // WatcherSpec defines the desired state of Watcher type WatcherSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -28,7 +33,6 @@ type WatcherSpec struct { WatcherTemplate `json:",inline"` - // +kubebuilder:validation:Required WatcherImages `json:",inline"` } @@ -40,6 +44,9 @@ type WatcherStatus struct { // ServiceID - The ID of the watcher service registered in keystone ServiceID string `json:"serviceID,omitempty"` + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + // ObservedGeneration - the most recent generation observed for this // service. If the observed generation is less than the spec generation, // then the controller has not processed the latest changes injected by diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 9c737b5..c4cbf23 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -450,6 +450,13 @@ func (in *WatcherStatus) DeepCopyInto(out *WatcherStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherStatus. diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index b27f85a..aa5afc5 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -70,6 +70,11 @@ spec: password from the Secret type: string type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean secret: description: Secret containing all passwords / keys needed type: string diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index 88174e6..ed6213c 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -75,6 +75,11 @@ spec: password from the Secret type: string type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean rabbitMqClusterName: default: rabbitmq description: |- @@ -142,6 +147,11 @@ spec: - type type: object type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object observedGeneration: description: |- ObservedGeneration - the most recent generation observed for this diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e05c9a6..fb60287 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -27,6 +27,18 @@ rules: - patch - update - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/watcher_controller.go b/controllers/watcher_controller.go index d5138b9..dc1979a 100644 --- a/controllers/watcher_controller.go +++ b/controllers/watcher_controller.go @@ -23,6 +23,7 @@ import ( "time" "github.com/go-logr/logr" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/types" @@ -34,7 +35,10 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/job" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -77,6 +81,7 @@ func (r *WatcherReconciler) GetLogger(ctx context.Context) logr.Logger { //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch +//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; // service account, role, rolebinding //+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch @@ -224,6 +229,21 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return ctrl.Result{}, errors.New("error retrieving required data from secret") } + hashTransporturl, _, transporturlSecret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: transportURL.Status.SecretName}, + []string{ + TransportURLSelector, + }, + helper.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if err != nil || hashTransporturl == "" { + // Empty hash means that there is some problem retrieving the key from the secret + return ctrl.Result{}, errors.New("error retrieving required data from transporturl secret") + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) // End of Input Ready check @@ -237,6 +257,31 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re // End of Keystone service creation + // Generate config for dbsync + configVars := make(map[string]env.Setter) + + err = r.generateServiceConfigDBSync(ctx, instance, db, &transporturlSecret, helper, &configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + // End of config generation for dbsync + + ctrlResult, err := r.ensureDBSync(ctx, helper, instance, serviceLabels) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // // remove finalizers from unused MariaDBAccount records // this assumes all database-depedendent deployments are up and // running with current database account info @@ -268,6 +313,11 @@ func (r *WatcherReconciler) initStatus(instance *watcherv1beta1.Watcher) error { // Update the lastObserved generation before evaluating conditions instance.Status.ObservedGeneration = instance.Generation + // initialize .Status.Hash + if instance.Status.Hash == nil { + instance.Status.Hash = make(map[string]string) + } + return nil } @@ -308,6 +358,14 @@ func (r *WatcherReconciler) initConditions(instance *watcherv1beta1.Watcher) err condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), + condition.UnknownCondition( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition( + condition.DBSyncReadyCondition, + condition.InitReason, + condition.DBSyncReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -544,6 +602,87 @@ func (r *WatcherReconciler) ensureKeystoneSvc( return ctrlResult, nil } +func (r *WatcherReconciler) generateServiceConfigDBSync( + ctx context.Context, + instance *watcherv1beta1.Watcher, + db *mariadbv1.Database, + transporturlSecret *corev1.Secret, + helper *helper.Helper, + envVars *map[string]env.Setter, +) error { + Log := r.GetLogger(ctx) + Log.Info("generateServiceConfigs - reconciling config for Watcher CR") + + customData := map[string]string{} + + labels := labels.GetLabels(instance, labels.GetGroupLabel(watcher.ServiceName), map[string]string{}) + databaseAccount := db.GetAccount() + databaseSecret := db.GetSecret() + templateParameters := map[string]interface{}{ + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?charset=utf8", + databaseAccount.Spec.UserName, + string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]), + db.GetDatabaseHostname(), + watcher.DatabaseName, + ), + "TransportURL": string(transporturlSecret.Data[TransportURLSelector]), + } + + return GenerateConfigsGeneric(ctx, helper, instance, envVars, templateParameters, customData, labels, false) +} + +func (r *WatcherReconciler) ensureDBSync( + ctx context.Context, + h *helper.Helper, + instance *watcherv1beta1.Watcher, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling the Keystone Service for '%s'", instance.Name)) + + // So far we are not using Service Annotations + dbSyncHash := instance.Status.Hash[watcherv1beta1.DbSyncHash] + jobDef := watcher.DbSyncJob(instance, serviceLabels, nil) + + dbSyncjob := job.NewJob( + jobDef, + watcherv1beta1.DbSyncHash, + instance.Spec.PreserveJobs, + time.Duration(5)*time.Second, + dbSyncHash, + ) + + ctrlResult, err := dbSyncjob.DoJob( + ctx, + h, + ) + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[watcherv1beta1.DbSyncHash] = dbSyncjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[watcherv1beta1.DbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + return ctrlResult, nil +} + func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watcherv1beta1.Watcher, helper *helper.Helper) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconcile Service '%s' delete started", instance.Name)) @@ -595,5 +734,6 @@ func (r *WatcherReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&batchv1.Job{}). Complete(r) } diff --git a/hack/crd-schema-checker.sh b/hack/crd-schema-checker.sh index 191e60b..9881695 100755 --- a/hack/crd-schema-checker.sh +++ b/hack/crd-schema-checker.sh @@ -17,6 +17,6 @@ for crd in config/crd/bases/*.yaml; do git show "$BASE_REF:$crd" > "$TMP_DIR/$crd" $CHECKER check-manifests \ --existing-crd-filename="$TMP_DIR/$crd" \ - --disabled-validators="NoFieldRemoval,NoNewRequiredFields,ConditionsMustHaveProperSSATags,ListsMustHaveSSATags" \ + --disabled-validators="NoFieldRemoval,NoNewRequiredFields,ConditionsMustHaveProperSSATags,ListsMustHaveSSATags,NoBools,NoMaps" \ --new-crd-filename="$crd" done diff --git a/pkg/watcher/constants.go b/pkg/watcher/constants.go index 3b20cb0..979cbf3 100644 --- a/pkg/watcher/constants.go +++ b/pkg/watcher/constants.go @@ -16,4 +16,13 @@ const ( // DatabaseCRName - name of the CR used to create the Watcher database DatabaseCRName = "watcher" + + // DefaultsConfigFileName - File name with default configuration + DefaultsConfigFileName = "00-default.conf" + + // CustomConfigFileName - File name with custom configuration + CustomConfigFileName = "01-custom.conf" + + // LogVolume is the default logVolume name used to mount logs + LogVolume = "logs" ) diff --git a/pkg/watcher/dbsync.go b/pkg/watcher/dbsync.go new file mode 100644 index 0000000..c5dbe7d --- /dev/null +++ b/pkg/watcher/dbsync.go @@ -0,0 +1,108 @@ +package watcher + +import ( + watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" + + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // DBSyncCommand - + DBSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// DbSyncJob func +func DbSyncJob(instance *watcherv1beta1.Watcher, labels map[string]string, annotations map[string]string) *batchv1.Job { + secretNames := []string{} + var config0644AccessMode int32 = 0644 + + // Unlike the individual Watcher services, the DbSyncJob doesn't need a + // secret that contains all of the config snippets required by every + // service, The two snippet files that it does need (DefaultsConfigFileName + // and CustomConfigFileName) can be extracted from the top-level watcher + // config-data secret. + dbSyncVolume := []corev1.Volume{ + { + Name: "db-sync-config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + Items: []corev1.KeyToPath{ + { + Key: DefaultsConfigFileName, + Path: DefaultsConfigFileName, + }, + }, + }, + }, + }, + } + + dbSyncMounts := []corev1.VolumeMount{ + { + Name: "db-sync-config-data", + MountPath: "/etc/watcher/watcher.conf.d", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "watcher-dbsync-config.json", + ReadOnly: true, + }, + } + + args := []string{"-c", DBSyncCommand} + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-db-sync", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: instance.Name + "-db-sync", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.APIContainerImageURL, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: append(GetVolumeMounts(secretNames), + dbSyncMounts...), + }, + }, + }, + }, + }, + } + + job.Spec.Template.Spec.Volumes = append(GetVolumes( + instance.Name, + secretNames), + dbSyncVolume..., + ) + + return job +} diff --git a/pkg/watcher/volumes.go b/pkg/watcher/volumes.go new file mode 100644 index 0000000..ec151ee --- /dev/null +++ b/pkg/watcher/volumes.go @@ -0,0 +1,102 @@ +package watcher + +import ( + "strconv" + + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - service volumes +func GetVolumes(name string, secretNames []string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + vm := []corev1.Volume{ + { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + } + + secretConfig, _ := GetConfigSecretVolumes(secretNames) + vm = append(vm, secretConfig...) + return vm +} + +// GetVolumeMounts - general VolumeMounts +func GetVolumeMounts(secretNames []string) []corev1.VolumeMount { + + vm := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/var/lib/config-data/default", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, + } + + _, secretConfig := GetConfigSecretVolumes(secretNames) + vm = append(vm, secretConfig...) + return vm +} + +// GetConfigSecretVolumes - Returns a list of volumes associated with a list of Secret names +func GetConfigSecretVolumes(secretNames []string) ([]corev1.Volume, []corev1.VolumeMount) { + var config0640AccessMode int32 = 0640 + secretVolumes := []corev1.Volume{} + secretMounts := []corev1.VolumeMount{} + + for idx, secretName := range secretNames { + secretVol := corev1.Volume{ + Name: secretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + DefaultMode: &config0640AccessMode, + }, + }, + } + secretMount := corev1.VolumeMount{ + Name: secretName, + // Each secret needs its own MountPath + MountPath: "/var/lib/config-data/secret-" + strconv.Itoa(idx), + ReadOnly: true, + } + secretVolumes = append(secretVolumes, secretVol) + secretMounts = append(secretMounts, secretMount) + } + + return secretVolumes, secretMounts +} + +// GetLogVolumeMount - Returns the VolumeMount used for logging purposes +func GetLogVolumeMount() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: LogVolume, + MountPath: "/var/log/watcher", + ReadOnly: false, + }, + } +} + +// GetLogVolume - Returns the Volume used for logging purposes +func GetLogVolume() []corev1.Volume { + return []corev1.Volume{ + { + Name: LogVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, + }, + }, + } +} diff --git a/templates/watcher/config/00-default.conf b/templates/watcher/config/00-default.conf index 8bf3025..5b73747 100644 --- a/templates/watcher/config/00-default.conf +++ b/templates/watcher/config/00-default.conf @@ -1,6 +1,8 @@ [DEFAULT] state_path = /var/lib/watcher +{{ if (index . "TransportURL") }} transport_url = {{ .TransportURL }} +{{ end }} control_exchange = watcher debug = True @@ -13,8 +15,11 @@ policy_file = /etc/watcher/policy.yaml.sample [oslo_messaging_notifications] driver = messagingv2 +{{ if (index . "KeystoneAuthURL") }} [keystone_authtoken] +{{ if (index . "MemcachedServers") }} memcached_servers = {{ .MemcachedServers }} +{{ end }} # TODO jgilaber implement handling this option when we add tls support # cafile = /var/lib/ca-bundle.pem project_domain_name = Default @@ -25,7 +30,9 @@ username = {{ .ServiceUser }} auth_url = {{ .KeystoneAuthURL }} interface = internal auth_type = password +{{ end }} +{{ if (index . "KeystoneAuthURL") }} [watcher_clients_auth] # TODO jgilaber implement handling this option when we add tls support # cafile = /var/lib/ca-bundle.pem @@ -37,6 +44,8 @@ username = {{ .ServiceUser }} auth_url = {{ .KeystoneAuthURL }} interface = internal auth_type = password +{{ end }} + [oslo_concurrency] lock_path = /var/lib/watcher/tmp @@ -44,5 +53,7 @@ lock_path = /var/lib/watcher/tmp [watcher_datasources] datasources = ceilometer +{{ if (index . "MemcachedServers") }} [cache] memcached_servers = {{ .MemcachedServers }} +{{ end }} diff --git a/templates/watcher/config/watcher-dbsync-config.json b/templates/watcher/config/watcher-dbsync-config.json new file mode 100644 index 0000000..a088412 --- /dev/null +++ b/templates/watcher/config/watcher-dbsync-config.json @@ -0,0 +1,3 @@ +{ + "command": "watcher-db-manage upgrade" +} diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index 5f09642..928255e 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -36,6 +36,16 @@ func GetDefaultWatcherSpec() map[string]interface{} { } } +// Second Watcher Spec to test proper parameters substitution +func GetNonDefaultWatcherSpec() map[string]interface{} { + return map[string]interface{}{ + "secret": SecretName, + "preserveJobs": true, + "databaseInstance": "fakeopenstack", + "serviceUser": "fakeuser", + } +} + func GetDefaultWatcherAPISpec() map[string]interface{} { return map[string]interface{}{ "databaseInstance": "openstack", diff --git a/tests/functional/watcher_controller_test.go b/tests/functional/watcher_controller_test.go index 835cecd..87b737d 100644 --- a/tests/functional/watcher_controller_test.go +++ b/tests/functional/watcher_controller_test.go @@ -11,6 +11,7 @@ import ( rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1beta1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" @@ -45,12 +46,14 @@ var _ = Describe("Watcher controller with minimal spec values", func() { Expect(Watcher.Spec.PasswordSelectors).Should(Equal(watcherv1beta1.PasswordSelector{Service: "WatcherPassword"})) Expect(Watcher.Spec.RabbitMqClusterName).Should(Equal("rabbitmq")) Expect(Watcher.Spec.ServiceUser).Should(Equal("watcher")) + Expect(Watcher.Spec.PreserveJobs).Should(BeFalse()) }) It("should have the Status fields initialized", func() { Watcher := GetWatcher(watcherTest.Instance) Expect(Watcher.Status.ObservedGeneration).To(Equal(int64(0))) Expect(Watcher.Status.ServiceID).Should(Equal("")) + Expect(Watcher.Status.Hash).Should(BeEmpty()) }) It("It has the expected container image defaults", func() { @@ -84,6 +87,7 @@ var _ = Describe("Watcher controller", func() { Expect(Watcher.Spec.ServiceUser).Should(Equal("watcher")) Expect(Watcher.Spec.Secret).Should(Equal("test-osp-secret")) Expect(Watcher.Spec.RabbitMqClusterName).Should(Equal("rabbitmq")) + Expect(Watcher.Spec.PreserveJobs).Should(BeFalse()) }) It("should have the Status fields initialized", func() { @@ -232,6 +236,8 @@ var _ = Describe("Watcher controller", func() { // did its job and registered the watcher service keystone.SimulateKeystoneServiceReady(watcherTest.KeystoneServiceName) + // Simulate dbsync success + th.SimulateJobSuccess(watcherTest.WatcherDBSync) // We validate the full Watcher CR readiness status here // DB Ready th.ExpectCondition( @@ -277,6 +283,14 @@ var _ = Describe("Watcher controller", func() { corev1.ConditionTrue, ) + // DBSync execution + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.DBSyncReadyCondition, + corev1.ConditionTrue, + ) + // Global status Ready th.ExpectCondition( watcherTest.Instance, @@ -295,6 +309,10 @@ var _ = Describe("Watcher controller", func() { Expect(ksrvList.Items).ToNot(BeEmpty()) Expect(ksrvList.Items[0].Status.Conditions).ToNot(BeNil()) + // status.hash['dbsync'] should be populated when dbsync is successful + Watcher := GetWatcher(watcherTest.Instance) + Expect(Watcher.Status.Hash[watcherv1beta1.DbSyncHash]).ShouldNot(BeNil()) + }) It("Should fail to register watcher service to keystone when has not the expected secret", func() { @@ -470,4 +488,136 @@ var _ = Describe("Watcher controller", func() { Expect(Watcher.Spec.ApplierContainerImageURL).To(Equal("watcher-applier-custom-image-env")) }) }) + When("Watcher with non-default values are created", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, GetNonDefaultWatcherSpec())) + DeferCleanup(k8sClient.Delete, ctx, CreateWatcherMessageBusSecret(watcherTest.Instance.Namespace, "rabbitmq-secret")) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + watcherTest.Instance.Namespace, + GetWatcher(watcherTest.Instance).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + }) + + It("should have the Spec fields with the expected values", func() { + Watcher := GetWatcher(watcherTest.Instance) + Expect(Watcher.Spec.DatabaseInstance).Should(Equal("fakeopenstack")) + Expect(Watcher.Spec.DatabaseAccount).Should(Equal("watcher")) + Expect(Watcher.Spec.ServiceUser).Should(Equal("fakeuser")) + Expect(Watcher.Spec.Secret).Should(Equal("test-osp-secret")) + Expect(Watcher.Spec.PreserveJobs).Should(BeTrue()) + Expect(Watcher.Spec.RabbitMqClusterName).Should(Equal("rabbitmq")) + }) + + It("Should create watcher service with custom values", func() { + mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount) + mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName) + infra.SimulateTransportURLReady(watcherTest.WatcherTransportURL) + DeferCleanup( + k8sClient.Delete, ctx, th.CreateSecret( + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: SecretName}, + map[string][]byte{ + "WatcherPassword": []byte("password"), + }, + )) + + // simulate that it becomes ready i.e. the keystone-operator + // did its job and registered the watcher service + keystone.SimulateKeystoneServiceReady(watcherTest.KeystoneServiceName) + + // Simulate dbsync success + th.SimulateJobSuccess(watcherTest.WatcherDBSync) + // We validate the full Watcher CR readiness status here + // DB Ready + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.DBReadyCondition, + corev1.ConditionTrue, + ) + // RabbitMQ Ready + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + watcherv1beta1.WatcherRabbitMQTransportURLReadyCondition, + corev1.ConditionTrue, + ) + // Input Ready (secrets) + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.InputReadyCondition, + corev1.ConditionTrue, + ) + // Keystone Service Ready + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.KeystoneServiceReadyCondition, + corev1.ConditionTrue, + ) + + // Service Account and Role Ready + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.ServiceAccountReadyCondition, + corev1.ConditionTrue, + ) + + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.RoleReadyCondition, + corev1.ConditionTrue, + ) + + // DBSync execution + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.DBSyncReadyCondition, + corev1.ConditionTrue, + ) + + // Global status Ready + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + // assert that the MariaDBDatabase is created in non-default Database + mariadbList := &mariadbv1.MariaDBDatabaseList{} + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", "watcher"), + Namespace: watcherTest.Instance.Namespace, + } + _ = th.K8sClient.List(ctx, mariadbList, listOpts) + // Check custom ServiceUser + Expect(mariadbList.Items[0].Labels["dbName"]).To(Equal("fakeopenstack")) + + // assert that the KeystoneService for watcher is created + ksrvList := &keystonev1beta1.KeystoneServiceList{} + listOpts = &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", "watcher"), + Namespace: watcherTest.Instance.Namespace, + } + _ = th.K8sClient.List(ctx, ksrvList, listOpts) + // Check custom ServiceUser + Expect(ksrvList.Items[0].Spec.ServiceUser).To(Equal("fakeuser")) + + // status.hash['dbsync'] should be populated when dbsync is successful + Watcher := GetWatcher(watcherTest.Instance) + Expect(Watcher.Status.Hash[watcherv1beta1.DbSyncHash]).ShouldNot(BeNil()) + + }) + }) + }) diff --git a/tests/functional/watcher_test_data.go b/tests/functional/watcher_test_data.go index bdd5cd5..f9e35ad 100644 --- a/tests/functional/watcher_test_data.go +++ b/tests/functional/watcher_test_data.go @@ -45,6 +45,7 @@ type WatcherTestData struct { ServiceAccountName types.NamespacedName RoleName types.NamespacedName RoleBindingName types.NamespacedName + WatcherDBSync types.NamespacedName } // GetWatcherTestData is a function that initialize the WatcherTestData @@ -105,5 +106,9 @@ func GetWatcherTestData(watcherName types.NamespacedName) WatcherTestData { Namespace: watcherName.Namespace, Name: "watcher-" + watcherName.Name + "-rolebinding", }, + WatcherDBSync: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: fmt.Sprintf("%s-db-sync", watcherName.Name), + }, } }