From 21433a762a5b6a54c14aab2af22d21bab4343d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 2 Apr 2020 15:32:59 -0300 Subject: [PATCH 01/47] Enable proxy_cache directive to enable cache on default location --- internal/pkg/rpaas/nginx/configuration_render.go | 4 ++++ internal/pkg/rpaas/nginx/configuration_render_test.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/internal/pkg/rpaas/nginx/configuration_render.go b/internal/pkg/rpaas/nginx/configuration_render.go index 0909e4b94..6aadf53b9 100644 --- a/internal/pkg/rpaas/nginx/configuration_render.go +++ b/internal/pkg/rpaas/nginx/configuration_render.go @@ -311,6 +311,10 @@ http { {{- end }} {{- end }} + {{- if boolValue $config.CacheEnabled }} + proxy_cache rpaas; + {{- end }} + location = /_nginx_healthcheck { {{- if boolValue $config.VTSEnabled }} vhost_traffic_status_bypass_limit on; diff --git a/internal/pkg/rpaas/nginx/configuration_render_test.go b/internal/pkg/rpaas/nginx/configuration_render_test.go index 2b32ed546..8108c34fc 100644 --- a/internal/pkg/rpaas/nginx/configuration_render_test.go +++ b/internal/pkg/rpaas/nginx/configuration_render_test.go @@ -83,6 +83,8 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { \s+proxy_cache_purge rpaas \$1\$is_args\$args; \s+} \s+}`, result) + assert.Regexp(t, `proxy_cache rpaas;`, result) + }, }, { From 024d48a36ac160e1a72a5e19dde5e2f62abc9209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 3 Apr 2020 11:03:39 -0300 Subject: [PATCH 02/47] Create PVC when CacheHeater is enabled --- .../extensions/v1alpha1/rpaasplan_types.go | 2 + .../rpaasinstance/rpaasinstance_controller.go | 87 +++++++++++++++++++ .../rpaasinstance_controller_test.go | 54 ++++++++++++ 3 files changed, 143 insertions(+) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 6eed4ebc2..e61bb6770 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -66,6 +66,8 @@ type NginxConfig struct { CacheSize string `json:"cacheSize,omitempty"` CacheZoneSize string `json:"cacheZoneSize,omitempty"` + CacheHeaterEnabled bool `json:"cacheHeaterEnabled"` + HTTPListenOptions string `json:"httpListenOptions,omitempty"` HTTPSListenOptions string `json:"httpsListenOptions,omitempty"` diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 5d540391f..05f2f00de 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" k8sErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" k8sResources "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -81,6 +82,14 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } + err = c.Watch(&source.Kind{Type: &corev1.PersistentVolume{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &extensionsv1alpha1.RpaasInstance{}, + }) + if err != nil { + return err + } + err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: &extensionsv1alpha1.RpaasInstance{}, @@ -184,6 +193,16 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile return reconcile.Result{}, err } + if plan.Spec.Config.CacheHeaterEnabled { + if err := r.reconcileCacheHeaterVolume(instance); err != nil { + return reconcile.Result{}, err + } + } else { + if err := r.destroyCacheHeaterVolume(instance); err != nil { + return reconcile.Result{}, err + } + } + if err = r.reconcileHPA(ctx, *instance, *nginx); err != nil { return reconcile.Result{}, err } @@ -404,6 +423,74 @@ func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) erro return err } +func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { + pvcName := instance.Name + "-heater-volume" + + pvc := &corev1.PersistentVolumeClaim{} + err := r.client.Get(context.TODO(), types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) + isNotFound := k8sErrors.IsNotFound(err) + if err != nil && !isNotFound { + return err + } else if !isNotFound { + logrus.Infof("PersistentVolumeClaim %s is found, skipping creation", pvcName) + return nil + } + + storageSize, err := resource.ParseQuantity("1Gi") + if err != nil { + return err + } + volumeMode := corev1.PersistentVolumeFilesystem + pvc = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: instance.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(instance, schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "RpaasInstance", + }), + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + VolumeMode: &volumeMode, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "storage": storageSize, + }, + }, + }, + } + if err := r.client.Create(context.TODO(), pvc); err != nil { + return err + } + return nil +} + +func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { + pvcName := instance.Name + "-heater-volume" + + pvc := &corev1.PersistentVolumeClaim{} + err := r.client.Get(context.TODO(), types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) + isNotFound := k8sErrors.IsNotFound(err) + if err != nil && !isNotFound { + return err + } else if isNotFound { + logrus.Infof("PersistentVolumeClaim %s is not found, skipping destruction", pvcName) + return nil + } + + return r.client.Delete(context.TODO(), pvc) +} + func (r *ReconcileRpaasInstance) renderTemplate(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) (string, error) { blocks, err := r.getConfigurationBlocks(instance, plan) if err != nil { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index fcfd0071a..0cbe80842 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -655,6 +655,60 @@ func Test_reconcileHPA(t *testing.T) { } } +func Test_reconcileHeaterVolume(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + err := reconciler.reconcileCacheHeaterVolume(instance1) + require.NoError(t, err) + + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.NoError(t, err) + + assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Kind, "RpaasInstance") + assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Name, instance1.Name) + assert.Equal(t, pvc.Spec.AccessModes, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}) +} + +func Test_destroyHeaterVolume(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "instance-1-heater-volume", + Namespace: "default", + }, + } + resources := []runtime.Object{pvc} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + err := reconciler.destroyCacheHeaterVolume(instance1) + require.NoError(t, err) + + pvc = &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.True(t, k8sErrors.IsNotFound(err)) +} + func int32Ptr(n int32) *int32 { return &n } From eeec773d7ce8b6c7662f359ee8925f682feeb412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 3 Apr 2020 16:10:49 -0300 Subject: [PATCH 03/47] Add support to specify storageClassName for CacheHeater --- pkg/apis/extensions/v1alpha1/rpaasplan_types.go | 7 ++++++- .../rpaasinstance/rpaasinstance_controller.go | 11 ++++++----- .../rpaasinstance_controller_test.go | 15 +++++++++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index e61bb6770..6d93de946 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -66,7 +66,8 @@ type NginxConfig struct { CacheSize string `json:"cacheSize,omitempty"` CacheZoneSize string `json:"cacheZoneSize,omitempty"` - CacheHeaterEnabled bool `json:"cacheHeaterEnabled"` + CacheHeaterEnabled bool `json:"cacheHeaterEnabled"` + CacheHeaterStorage *CacheHeaterStorage `json:"cacheHeaterStorage"` HTTPListenOptions string `json:"httpListenOptions,omitempty"` HTTPSListenOptions string `json:"httpsListenOptions,omitempty"` @@ -83,6 +84,10 @@ type NginxConfig struct { WorkerConnections int `json:"workerConnections,omitempty"` } +type CacheHeaterStorage struct { + StorageClassName *string `json:"storageClassName"` +} + func Bool(v bool) *bool { return &v } diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 05f2f00de..23a6ca3f8 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -194,11 +194,11 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile } if plan.Spec.Config.CacheHeaterEnabled { - if err := r.reconcileCacheHeaterVolume(instance); err != nil { + if err := r.reconcileCacheHeaterVolume(instance, plan.Spec.Config.CacheHeaterStorage); err != nil { return reconcile.Result{}, err } } else { - if err := r.destroyCacheHeaterVolume(instance); err != nil { + if err := r.destroyCacheHeaterVolume(instance, plan.Spec.Config.CacheHeaterStorage); err != nil { return reconcile.Result{}, err } } @@ -423,7 +423,7 @@ func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) erro return err } -func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { +func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, storageConfig *v1alpha1.CacheHeaterStorage) error { pvcName := instance.Name + "-heater-volume" pvc := &corev1.PersistentVolumeClaim{} @@ -461,7 +461,8 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R AccessModes: []corev1.PersistentVolumeAccessMode{ corev1.ReadWriteMany, }, - VolumeMode: &volumeMode, + VolumeMode: &volumeMode, + StorageClassName: storageConfig.StorageClassName, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ "storage": storageSize, @@ -475,7 +476,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R return nil } -func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { +func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance, storageConfig *v1alpha1.CacheHeaterStorage) error { pvcName := instance.Name + "-heater-volume" pvc := &corev1.PersistentVolumeClaim{} diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 0cbe80842..443b5cac1 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -669,7 +669,10 @@ func Test_reconcileHeaterVolume(t *testing.T) { scheme: newScheme(), } - err := reconciler.reconcileCacheHeaterVolume(instance1) + storageConfig := &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + } + err := reconciler.reconcileCacheHeaterVolume(instance1, storageConfig) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -678,6 +681,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Kind, "RpaasInstance") assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Name, instance1.Name) + assert.Equal(t, pvc.Spec.StorageClassName, strPtr("my-storage-class")) assert.Equal(t, pvc.Spec.AccessModes, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}) } @@ -691,6 +695,9 @@ func Test_destroyHeaterVolume(t *testing.T) { Namespace: "default", }, } + storageConfig := &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + } resources := []runtime.Object{pvc} scheme := newScheme() corev1.AddToScheme(scheme) @@ -701,7 +708,7 @@ func Test_destroyHeaterVolume(t *testing.T) { scheme: newScheme(), } - err := reconciler.destroyCacheHeaterVolume(instance1) + err := reconciler.destroyCacheHeaterVolume(instance1, storageConfig) require.NoError(t, err) pvc = &corev1.PersistentVolumeClaim{} @@ -713,6 +720,10 @@ func int32Ptr(n int32) *int32 { return &n } +func strPtr(s string) *string { + return &s +} + func newEmptyRpaasInstance() *v1alpha1.RpaasInstance { return &v1alpha1.RpaasInstance{ TypeMeta: metav1.TypeMeta{ From 0a6f80a9ec130a1155a67e8bb7dca9412f0a7ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 3 Apr 2020 17:48:09 -0300 Subject: [PATCH 04/47] Attach PersistentVolumeClaim on Nginx instance --- .../rpaasinstance/rpaasinstance_controller.go | 21 +++++++++- .../rpaasinstance_controller_test.go | 41 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 23a6ca3f8..778ee85c7 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -441,6 +441,10 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R return err } volumeMode := corev1.PersistentVolumeFilesystem + var storageClassName *string + if storageConfig != nil { + storageClassName = storageConfig.StorageClassName + } pvc = &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -462,7 +466,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R corev1.ReadWriteMany, }, VolumeMode: &volumeMode, - StorageClassName: storageConfig.StorageClassName, + StorageClassName: storageClassName, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ "storage": storageSize, @@ -625,7 +629,7 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config cacheConfig.Size = &cacheMaxSize } } - return &nginxv1alpha1.Nginx{ + n := &nginxv1alpha1.Nginx{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, Namespace: instance.Namespace, @@ -658,6 +662,19 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config Lifecycle: instance.Spec.Lifecycle, }, } + + if plan.Spec.Config.CacheHeaterEnabled { + n.Spec.PodTemplate.Volumes = append(n.Spec.PodTemplate.Volumes, corev1.Volume{ + Name: "cache-heater-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: instance.Name + "-heater-volume", + }, + }, + }) + } + + return n } func newHPA(instance v1alpha1.RpaasInstance, nginx nginxv1alpha1.Nginx) autoscalingv2beta2.HorizontalPodAutoscaler { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 443b5cac1..ef391a4d2 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) func Test_mergePlans(t *testing.T) { @@ -1248,3 +1249,43 @@ func TestReconcileNginx_reconcilePorts(t *testing.T) { }) } } + +func TestReconcile(t *testing.T) { + rpaas := &v1alpha1.RpaasInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-instance", + Namespace: "default", + }, + Spec: v1alpha1.RpaasInstanceSpec{ + PlanName: "my-plan", + }, + } + plan := &v1alpha1.RpaasPlan{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-plan", + Namespace: "default", + }, + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterEnabled: true, + }, + }, + } + resources := []runtime.Object{rpaas, plan} + scheme := newScheme() + corev1.AddToScheme(scheme) + client := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: client, + scheme: scheme, + } + result, err := reconciler.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "my-instance"}}) + require.NoError(t, err) + + nginx := &nginxv1alpha1.Nginx{} + err = client.Get(context.TODO(), types.NamespacedName{Name: rpaas.Name, Namespace: rpaas.Namespace}, nginx) + require.NoError(t, err) + + assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].Name, "cache-heater-volume") + assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instanceheater-volume"}) +} From d5f20da445457730dc3ceeceb085b662c37405ce Mon Sep 17 00:00:00 2001 From: Bernardo Lins Date: Fri, 3 Apr 2020 19:05:33 -0300 Subject: [PATCH 05/47] feat(rpaas): add field to configure cache heater storage size --- .../extensions/v1alpha1/rpaasplan_types.go | 1 + .../rpaasinstance/rpaasinstance_controller.go | 39 +++++---- .../rpaasinstance_controller_test.go | 86 ++++++++++++++++++- 3 files changed, 107 insertions(+), 19 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 6d93de946..e4b5f55da 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -86,6 +86,7 @@ type NginxConfig struct { type CacheHeaterStorage struct { StorageClassName *string `json:"storageClassName"` + StorageSize string `json:"storageSize"` } func Bool(v bool) *bool { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 778ee85c7..016a2ab19 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -194,7 +194,7 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile } if plan.Spec.Config.CacheHeaterEnabled { - if err := r.reconcileCacheHeaterVolume(instance, plan.Spec.Config.CacheHeaterStorage); err != nil { + if err := r.reconcileCacheHeaterVolume(instance, plan); err != nil { return reconcile.Result{}, err } } else { @@ -423,7 +423,7 @@ func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) erro return err } -func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, storageConfig *v1alpha1.CacheHeaterStorage) error { +func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { pvcName := instance.Name + "-heater-volume" pvc := &corev1.PersistentVolumeClaim{} @@ -436,15 +436,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R return nil } - storageSize, err := resource.ParseQuantity("1Gi") - if err != nil { - return err - } volumeMode := corev1.PersistentVolumeFilesystem - var storageClassName *string - if storageConfig != nil { - storageClassName = storageConfig.StorageClassName - } pvc = &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -466,14 +458,29 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R corev1.ReadWriteMany, }, VolumeMode: &volumeMode, - StorageClassName: storageClassName, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "storage": storageSize, - }, - }, + StorageClassName: plan.Spec.Config.CacheHeaterStorage.StorageClassName, }, } + + storageSize := plan.Spec.Config.CacheSize + if cacheHeaterStorage := plan.Spec.Config.CacheHeaterStorage; cacheHeaterStorage != nil { + if cacheHeaterStorage.StorageSize != "" { + storageSize = cacheHeaterStorage.StorageSize + } + } + + if storageSize != "" { + parsedSize, err := resource.ParseQuantity(storageSize) + if err != nil { + return err + } + pvc.Spec.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "storage": parsedSize, + }, + } + } + if err := r.client.Create(context.TODO(), pvc); err != nil { return err } diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index ef391a4d2..63306fc32 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -670,10 +670,17 @@ func Test_reconcileHeaterVolume(t *testing.T) { scheme: newScheme(), } - storageConfig := &v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + }, + }, + }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, storageConfig) + + err := reconciler.reconcileCacheHeaterVolume(instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -686,6 +693,79 @@ func Test_reconcileHeaterVolume(t *testing.T) { assert.Equal(t, pvc.Spec.AccessModes, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}) } +func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheSize: "10Gi", + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + }, + }, + }, + } + + err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + require.NoError(t, err) + + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.NoError(t, err) + + parsedSize, err := resource.ParseQuantity("10Gi") + assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) +} + +func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheSize: "10Gi", + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + StorageSize: "100Gi", + }, + }, + }, + } + + err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + require.NoError(t, err) + + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.NoError(t, err) + + parsedSize, err := resource.ParseQuantity("100Gi") + assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) +} + func Test_destroyHeaterVolume(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" From a1f7f0aca834c452bbaf6f75d00c1a5f81c1ea72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 6 Apr 2020 10:10:27 -0300 Subject: [PATCH 06/47] Declare VolumeMount when cacheheater is enabled --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 5 +++++ .../rpaasinstance/rpaasinstance_controller_test.go | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 016a2ab19..88a4fb29e 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -679,6 +679,11 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config }, }, }) + + n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, corev1.VolumeMount{ + Name: "cache-heater-volume", + MountPath: "/var/cache/cache-heater", + }) } return n diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 63306fc32..5db31a680 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -1362,10 +1362,14 @@ func TestReconcile(t *testing.T) { result, err := reconciler.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "my-instance"}}) require.NoError(t, err) + assert.Equal(t, result, reconcile.Result{}) + nginx := &nginxv1alpha1.Nginx{} err = client.Get(context.TODO(), types.NamespacedName{Name: rpaas.Name, Namespace: rpaas.Namespace}, nginx) require.NoError(t, err) assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].Name, "cache-heater-volume") - assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instanceheater-volume"}) + assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instance-heater-volume"}) + assert.Equal(t, nginx.Spec.PodTemplate.VolumeMounts[0].Name, "cache-heater-volume") + assert.Equal(t, nginx.Spec.PodTemplate.VolumeMounts[0].MountPath, "/var/cache/cache-heater") } From 8ef1807e90d7ac2533efa966acad7f708070375e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 6 Apr 2020 14:48:19 -0300 Subject: [PATCH 07/47] Update nginx-operator dependency to fix tests --- go.mod | 2 +- go.sum | 4 ++-- .../rpaasinstance/rpaasinstance_controller_test.go | 3 +++ test/testdata/rpaas-full.yaml | 10 ++++++++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index ac10bd97f..4a3f7fca2 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.4.0 - github.com/tsuru/nginx-operator v0.5.0 + github.com/tsuru/nginx-operator v0.5.1 github.com/urfave/cli/v2 v2.0.0 github.com/willf/bitset v1.1.10 k8s.io/api v0.0.0 diff --git a/go.sum b/go.sum index 9c3d872ee..4bb6817f1 100644 --- a/go.sum +++ b/go.sum @@ -611,8 +611,8 @@ github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1C github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tsuru/config v0.0.0-20180418191556-87403ee7da02/go.mod h1:bilf/jr5AGl9raMFnhM9hsp+ms6PK752UF89ODoRHW4= -github.com/tsuru/nginx-operator v0.5.0 h1:oWSu/9WHm+wKbvJSZn0NebQZCw2R0R5HYgGNZcasp+w= -github.com/tsuru/nginx-operator v0.5.0/go.mod h1:hvXj+bM5dUsnOt/PQE1ktHnLxM1ucnhKM0zOdHZyzsg= +github.com/tsuru/nginx-operator v0.5.1 h1:vgLTB8om6wC91loqZcR5z1v5ONgLuIev1zzxG9WlHfk= +github.com/tsuru/nginx-operator v0.5.1/go.mod h1:hvXj+bM5dUsnOt/PQE1ktHnLxM1ucnhKM0zOdHZyzsg= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 5db31a680..2d8c8cdb2 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -1348,6 +1348,9 @@ func TestReconcile(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ CacheHeaterEnabled: true, + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + }, }, }, } diff --git a/test/testdata/rpaas-full.yaml b/test/testdata/rpaas-full.yaml index 2e740ef99..a6df0747e 100644 --- a/test/testdata/rpaas-full.yaml +++ b/test/testdata/rpaas-full.yaml @@ -3,8 +3,14 @@ kind: RpaasPlan metadata: name: basic spec: - image: tsuru/nginx-tsuru:1.15.0 - config: {} + image: tsuru/nginx-tsuru:1.16.1 + config: + cacheEnabled: true + cachePath: /var/cache/nginx + cacheZoneSize: 100m + cacheSize: 300m + + cacheHeaterEnabled: true resources: limits: memory: "128Mi" From 78d87335333d6224c2390364341f7ada03a54ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 6 Apr 2020 14:58:59 -0300 Subject: [PATCH 08/47] Fix golang lint --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 6 ++++-- .../rpaasinstance/rpaasinstance_controller_test.go | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 88a4fb29e..e27df4ced 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -194,11 +194,13 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile } if plan.Spec.Config.CacheHeaterEnabled { - if err := r.reconcileCacheHeaterVolume(instance, plan); err != nil { + err = r.reconcileCacheHeaterVolume(instance, plan) + if err != nil { return reconcile.Result{}, err } } else { - if err := r.destroyCacheHeaterVolume(instance, plan.Spec.Config.CacheHeaterStorage); err != nil { + err = r.destroyCacheHeaterVolume(instance, plan.Spec.Config.CacheHeaterStorage) + if err != nil { return reconcile.Result{}, err } } diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 2d8c8cdb2..272e704e7 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -726,6 +726,7 @@ func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { require.NoError(t, err) parsedSize, err := resource.ParseQuantity("10Gi") + require.NoError(t, err) assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) } @@ -763,6 +764,7 @@ func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { require.NoError(t, err) parsedSize, err := resource.ParseQuantity("100Gi") + require.NoError(t, err) assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) } From 7e15af1378646f564e7b0d93c90139d5138cf23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 6 Apr 2020 16:25:53 -0300 Subject: [PATCH 09/47] Fix Watch of PersistentVolumeClaim --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index e27df4ced..273a8073b 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -82,7 +82,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } - err = c.Watch(&source.Kind{Type: &corev1.PersistentVolume{}}, &handler.EnqueueRequestForOwner{ + err = c.Watch(&source.Kind{Type: &corev1.PersistentVolumeClaim{}}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: &extensionsv1alpha1.RpaasInstance{}, }) From fe40c7cc9455a80479c15c47d37ff5936694c740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 6 Apr 2020 19:03:46 -0300 Subject: [PATCH 10/47] Avoid crashes when miss CacheHeaterStorage --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 273a8073b..40ee4ee3d 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -438,6 +438,11 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R return nil } + cacheHeaterStorage := &v1alpha1.CacheHeaterStorage{} + if plan.Spec.Config.CacheHeaterStorage != nil { + cacheHeaterStorage = plan.Spec.Config.CacheHeaterStorage + } + volumeMode := corev1.PersistentVolumeFilesystem pvc = &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -460,7 +465,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R corev1.ReadWriteMany, }, VolumeMode: &volumeMode, - StorageClassName: plan.Spec.Config.CacheHeaterStorage.StorageClassName, + StorageClassName: cacheHeaterStorage.StorageClassName, }, } From 783d5d1a960832e0c86c75e63bed3663f79daa14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 7 Apr 2020 09:17:43 -0300 Subject: [PATCH 11/47] Disable cache heater for tests --- test/testdata/rpaas-full.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/testdata/rpaas-full.yaml b/test/testdata/rpaas-full.yaml index a6df0747e..91d9558f7 100644 --- a/test/testdata/rpaas-full.yaml +++ b/test/testdata/rpaas-full.yaml @@ -9,8 +9,7 @@ spec: cachePath: /var/cache/nginx cacheZoneSize: 100m cacheSize: 300m - - cacheHeaterEnabled: true + cacheHeaterEnabled: false resources: limits: memory: "128Mi" From fc2f4e754df855a770889f151165bcc137de8042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 7 Apr 2020 09:36:42 -0300 Subject: [PATCH 12/47] Fix tests of version of rpaas --- test/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration_test.go b/test/integration_test.go index 249100579..b590bcada 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -50,7 +50,7 @@ func Test_RpaasOperator(t *testing.T) { nginx, err := getReadyNginx("my-instance", namespaceName, 2, 1) require.NoError(t, err) assert.Equal(t, int32(2), *nginx.Spec.Replicas) - assert.Equal(t, "tsuru/nginx-tsuru:1.15.0", nginx.Spec.Image) + assert.Equal(t, "tsuru/nginx-tsuru:1.16.1", nginx.Spec.Image) assert.Equal(t, "/_nginx_healthcheck", nginx.Spec.HealthcheckPath) assert.Len(t, nginx.Status.Pods, 2) for _, podStatus := range nginx.Status.Pods { @@ -176,7 +176,7 @@ func Test_RpaasApi(t *testing.T) { require.NoError(t, err) require.NotNil(t, nginx) assert.Equal(t, int32(1), *nginx.Spec.Replicas) - assert.Equal(t, "tsuru/nginx-tsuru:1.15.0", nginx.Spec.Image) + assert.Equal(t, "tsuru/nginx-tsuru:1.16.1", nginx.Spec.Image) assert.Equal(t, "/_nginx_healthcheck", nginx.Spec.HealthcheckPath) nginxService := &corev1.Service{ From 5977020b495f783eedbcba3f750870e72ddc859a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 7 Apr 2020 09:54:15 -0300 Subject: [PATCH 13/47] Fix tests of version of rpaas[2] --- test/testdata/rpaasplan-basic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testdata/rpaasplan-basic.yaml b/test/testdata/rpaasplan-basic.yaml index f7aaad28a..d0c45ecd9 100644 --- a/test/testdata/rpaasplan-basic.yaml +++ b/test/testdata/rpaasplan-basic.yaml @@ -3,7 +3,7 @@ kind: RpaasPlan metadata: name: basic spec: - image: tsuru/nginx-tsuru:1.15.0 + image: tsuru/nginx-tsuru:1.16.1 config: {} resources: requests: From 4842cb7b00268c406910a029a5ff9f0c3363bd60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 7 Apr 2020 11:44:44 -0300 Subject: [PATCH 14/47] Update generated code by `operator-sdk` --- .../v1alpha1/zz_generated.deepcopy.go | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go index c17e14f74..adca8a43c 100644 --- a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go @@ -47,6 +47,27 @@ func (in *Bind) DeepCopy() *Bind { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheHeaterStorage) DeepCopyInto(out *CacheHeaterStorage) { + *out = *in + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheHeaterStorage. +func (in *CacheHeaterStorage) DeepCopy() *CacheHeaterStorage { + if in == nil { + return nil + } + out := new(CacheHeaterStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Location) DeepCopyInto(out *Location) { *out = *in @@ -92,6 +113,11 @@ func (in *NginxConfig) DeepCopyInto(out *NginxConfig) { *out = new(bool) **out = **in } + if in.CacheHeaterStorage != nil { + in, out := &in.CacheHeaterStorage, &out.CacheHeaterStorage + *out = new(CacheHeaterStorage) + (*in).DeepCopyInto(*out) + } if in.VTSEnabled != nil { in, out := &in.VTSEnabled, &out.VTSEnabled *out = new(bool) From d8e151e397d1ea35b2374e491181383ff59501d2 Mon Sep 17 00:00:00 2001 From: Bernardo Lins Date: Wed, 8 Apr 2020 15:26:53 -0300 Subject: [PATCH 15/47] feat(rpaas): add volume labels to CacheHeaterStorage --- .../extensions/v1alpha1/rpaasplan_types.go | 9 +++-- .../rpaasinstance/rpaasinstance_controller.go | 1 + .../rpaasinstance_controller_test.go | 40 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index e4b5f55da..015e87258 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -66,8 +66,8 @@ type NginxConfig struct { CacheSize string `json:"cacheSize,omitempty"` CacheZoneSize string `json:"cacheZoneSize,omitempty"` - CacheHeaterEnabled bool `json:"cacheHeaterEnabled"` - CacheHeaterStorage *CacheHeaterStorage `json:"cacheHeaterStorage"` + CacheHeaterEnabled bool `json:"cacheHeaterEnabled,omitempty"` + CacheHeaterStorage *CacheHeaterStorage `json:"cacheHeaterStorage,omitempty"` HTTPListenOptions string `json:"httpListenOptions,omitempty"` HTTPSListenOptions string `json:"httpsListenOptions,omitempty"` @@ -85,8 +85,9 @@ type NginxConfig struct { } type CacheHeaterStorage struct { - StorageClassName *string `json:"storageClassName"` - StorageSize string `json:"storageSize"` + StorageClassName *string `json:"storageClassName,omitempty"` + StorageSize string `json:"storageSize,omitempty"` + VolumeLabels map[string]string `json:"volumeLabels,omitempty"` } func Bool(v bool) *bool { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 40ee4ee3d..7d1282ebe 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -448,6 +448,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R ObjectMeta: metav1.ObjectMeta{ Name: pvcName, Namespace: instance.Namespace, + Labels: cacheHeaterStorage.VolumeLabels, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(instance, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 272e704e7..fc732ce67 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -693,6 +693,46 @@ func Test_reconcileHeaterVolume(t *testing.T) { assert.Equal(t, pvc.Spec.AccessModes, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}) } +func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + VolumeLabels: map[string]string{ + "some-label": "foo", + "other-label": "bar", + }, + }, + }, + }, + } + + err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + require.NoError(t, err) + + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.NoError(t, err) + + assert.Equal(t, 2, len(pvc.ObjectMeta.Labels)) + assert.Equal(t, "foo", pvc.ObjectMeta.Labels["some-label"]) + assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) +} + func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" From a67ab603cb390580491fa6a25d97ef3a4a03a233 Mon Sep 17 00:00:00 2001 From: Bernardo Lins Date: Tue, 14 Apr 2020 18:12:10 -0300 Subject: [PATCH 16/47] refactor: set team owner on the instance --- pkg/apis/extensions/v1alpha1/rpaasinstance.go | 26 ++++++++++++++++ .../extensions/v1alpha1/rpaasinstance_test.go | 31 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 pkg/apis/extensions/v1alpha1/rpaasinstance.go create mode 100644 pkg/apis/extensions/v1alpha1/rpaasinstance_test.go diff --git a/pkg/apis/extensions/v1alpha1/rpaasinstance.go b/pkg/apis/extensions/v1alpha1/rpaasinstance.go new file mode 100644 index 000000000..f242fa5bb --- /dev/null +++ b/pkg/apis/extensions/v1alpha1/rpaasinstance.go @@ -0,0 +1,26 @@ +package v1alpha1 + +const( + teamOwnerLabel = "rpaas.extensions.tsuru.io/team-owner" +) + +func (i *RpaasInstance) SetTeamOwner(team string) { + newLabels := map[string]string{teamOwnerLabel: team} + i.Labels = mergeMap(i.Labels, newLabels) + i.Annotations = mergeMap(i.Annotations, newLabels) + i.Spec.PodTemplate.Labels = mergeMap(i.Spec.PodTemplate.Labels, newLabels) +} + +func (i *RpaasInstance) TeamOwner() string { + return i.Labels[teamOwnerLabel] +} + +func mergeMap(a, b map[string]string) map[string]string { + if a == nil { + return b + } + for k, v := range b { + a[k] = v + } + return a +} diff --git a/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go b/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go new file mode 100644 index 000000000..8261151e5 --- /dev/null +++ b/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go @@ -0,0 +1,31 @@ +package v1alpha1 + +import( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_SetTeamOwner(t *testing.T) { + instance := &RpaasInstance{} + instance.SetTeamOwner("team-one") + expected := map[string]string{teamOwnerLabel: "team-one"} + assert.Equal(t, expected, instance.Labels) + assert.Equal(t, expected, instance.Annotations) + assert.Equal(t, expected, instance.Spec.PodTemplate.Labels) + + instance.SetTeamOwner("team-two") + expected = map[string]string{teamOwnerLabel: "team-two"} + assert.Equal(t, expected, instance.Labels) + assert.Equal(t, expected, instance.Annotations) + assert.Equal(t, expected, instance.Spec.PodTemplate.Labels) +} + +func Test_GetTeamOwner(t *testing.T) { + instance := &RpaasInstance{} + owner := instance.TeamOwner() + assert.Equal(t, "", owner) + instance.SetTeamOwner("team-one") + owner = instance.TeamOwner() + assert.Equal(t, "team-one", owner) +} From 59779c72670dc45b9bb1a65e65847c507546f299 Mon Sep 17 00:00:00 2001 From: Bernardo Lins Date: Tue, 14 Apr 2020 18:19:27 -0300 Subject: [PATCH 17/47] refactor(k8s): using instance SetTeamOwner --- internal/pkg/rpaas/k8s.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/internal/pkg/rpaas/k8s.go b/internal/pkg/rpaas/k8s.go index 6c0295dbd..8aa1c1002 100644 --- a/internal/pkg/rpaas/k8s.go +++ b/internal/pkg/rpaas/k8s.go @@ -112,7 +112,7 @@ func (m *k8sRpaasManager) CreateInstance(ctx context.Context, args CreateArgs) e } setDescription(instance, args.Description) - setTeamOwner(instance, args.Team) + instance.SetTeamOwner(args.Team) setTags(instance, args.Tags) setIP(instance, args.IP()) @@ -145,7 +145,7 @@ func (m *k8sRpaasManager) UpdateInstance(ctx context.Context, instanceName strin instance.Spec.Flavors = args.Flavors() setDescription(instance, args.Description) - setTeamOwner(instance, args.Team) + instance.SetTeamOwner(args.Team) setTags(instance, args.Tags) setIP(instance, args.IP()) @@ -1411,18 +1411,6 @@ func setTags(instance *v1alpha1.RpaasInstance, tags []string) { }) } -func setTeamOwner(instance *v1alpha1.RpaasInstance, team string) { - if instance == nil { - return - } - - newLabels := map[string]string{labelKey("team-owner"): team} - - instance.Annotations = mergeMap(instance.Annotations, newLabels) - instance.Labels = mergeMap(instance.Labels, newLabels) - instance.Spec.PodTemplate.Labels = mergeMap(instance.Spec.PodTemplate.Labels, newLabels) -} - func (m *k8sRpaasManager) GetInstanceInfo(ctx context.Context, instanceName string) (*clientTypes.InstanceInfo, error) { instance, err := m.GetInstance(ctx, instanceName) if err != nil { From 3428b520620fa0ec383c84caa9872624a249f7cb Mon Sep 17 00:00:00 2001 From: Bernardo Lins Date: Wed, 15 Apr 2020 11:27:03 -0300 Subject: [PATCH 18/47] feat(rpaasinstance): set team ower as a pvc label on cache heater --- .../rpaasinstance/rpaasinstance_controller.go | 8 ++++ .../rpaasinstance_controller_test.go | 43 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 7d1282ebe..cbeebecf6 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -45,6 +45,7 @@ const ( defaultConfigHistoryLimit = 10 defaultPortAllocationResource = "default" + volumeTeamLabel = "tsuru.io/volume-name" ) var log = logf.Log.WithName("controller_rpaasinstance") @@ -444,6 +445,13 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R } volumeMode := corev1.PersistentVolumeFilesystem + labels := map[string]string{} + if cacheHeaterStorage.VolumeLabels != nil { + labels = cacheHeaterStorage.VolumeLabels + } + if teamOwner := instance.TeamOwner(); teamOwner != "" { + labels[volumeTeamLabel] = teamOwner + } pvc = &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index fc732ce67..5a6981c3c 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -733,6 +733,49 @@ func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) } +func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + instance1.SetTeamOwner("team-one") + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + VolumeLabels: map[string]string{ + "some-label": "foo", + "other-label": "bar", + volumeTeamLabel: "another-team", + }, + }, + }, + }, + } + + err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + require.NoError(t, err) + + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.NoError(t, err) + + assert.Equal(t, 3, len(pvc.ObjectMeta.Labels)) + assert.Equal(t, "foo", pvc.ObjectMeta.Labels["some-label"]) + assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) + assert.Equal(t, "team-one", pvc.ObjectMeta.Labels[volumeTeamLabel]) +} + func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" From d6c65c985d5ff5de02f6a1973160ec1454c3e96c Mon Sep 17 00:00:00 2001 From: Bernardo Lins Date: Wed, 15 Apr 2020 16:37:15 -0300 Subject: [PATCH 19/47] feat(rpaasinstance): prioritize plan volume label instead of instance's --- .../rpaasinstance/rpaasinstance_controller.go | 9 ++-- .../rpaasinstance_controller_test.go | 44 ++++++++++++++++++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index cbeebecf6..b568749f9 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -443,20 +443,19 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R if plan.Spec.Config.CacheHeaterStorage != nil { cacheHeaterStorage = plan.Spec.Config.CacheHeaterStorage } - volumeMode := corev1.PersistentVolumeFilesystem labels := map[string]string{} - if cacheHeaterStorage.VolumeLabels != nil { - labels = cacheHeaterStorage.VolumeLabels - } if teamOwner := instance.TeamOwner(); teamOwner != "" { labels[volumeTeamLabel] = teamOwner + } + for k, v := range cacheHeaterStorage.VolumeLabels { + labels[k] = v } pvc = &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, Namespace: instance.Namespace, - Labels: cacheHeaterStorage.VolumeLabels, + Labels: labels, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(instance, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 5a6981c3c..62b77ab90 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -756,7 +756,6 @@ func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { VolumeLabels: map[string]string{ "some-label": "foo", "other-label": "bar", - volumeTeamLabel: "another-team", }, }, }, @@ -776,6 +775,49 @@ func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { assert.Equal(t, "team-one", pvc.ObjectMeta.Labels[volumeTeamLabel]) } +func Test_reconcileHeaterVolumeLabels(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + instance1.SetTeamOwner("team-one") + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + VolumeLabels: map[string]string{ + "some-label": "foo", + "other-label": "bar", + volumeTeamLabel: "another-team", + }, + }, + }, + }, + } + + err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + require.NoError(t, err) + + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + require.NoError(t, err) + + assert.Equal(t, 3, len(pvc.ObjectMeta.Labels)) + assert.Equal(t, "foo", pvc.ObjectMeta.Labels["some-label"]) + assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) + assert.Equal(t, "another-team", pvc.ObjectMeta.Labels[volumeTeamLabel]) +} + func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" From 6e81e429488c19c545b0d3667614460a28db7ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 16 Apr 2020 18:10:05 -0300 Subject: [PATCH 20/47] Add reconcile process to create CronJobs --- .../extensions/v1alpha1/rpaasplan_types.go | 10 ++ .../rpaasinstance/rpaasinstance_controller.go | 105 ++++++++++++++++-- .../rpaasinstance_controller_test.go | 30 ++++- 3 files changed, 132 insertions(+), 13 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 015e87258..599a6811b 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -68,6 +68,7 @@ type NginxConfig struct { CacheHeaterEnabled bool `json:"cacheHeaterEnabled,omitempty"` CacheHeaterStorage *CacheHeaterStorage `json:"cacheHeaterStorage,omitempty"` + CacheHeaterSync CacheHeaterSyncSpec `json:"cacheHeaterSync,omitempty"` HTTPListenOptions string `json:"httpListenOptions,omitempty"` HTTPSListenOptions string `json:"httpsListenOptions,omitempty"` @@ -84,6 +85,15 @@ type NginxConfig struct { WorkerConnections int `json:"workerConnections,omitempty"` } +type CacheHeaterSyncSpec struct { + // The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. + Schedule string `json:"schedule,omitempty"` + + // Container image used to sync the containers + // default is bitnami/kubectl:latest + Image string `json:"image,omitempty"` +} + type CacheHeaterStorage struct { StorageClassName *string `json:"storageClassName,omitempty"` StorageSize string `json:"storageSize,omitempty"` diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index b568749f9..333af6e3f 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -23,6 +23,8 @@ import ( "github.com/tsuru/rpaas-operator/pkg/util" "github.com/willf/bitset" autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + batchv1 "k8s.io/api/batch/v1" + batchv1beta1 "k8s.io/api/batch/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" k8sErrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,10 +44,12 @@ import ( ) const ( - defaultConfigHistoryLimit = 10 + defaultConfigHistoryLimit = 10 + defaultCacheHeaterCronImage = "bitnami/kubectl:latest" + defaultCacheHeaterSchedule = "* * * * *" defaultPortAllocationResource = "default" - volumeTeamLabel = "tsuru.io/volume-name" + volumeTeamLabel = "tsuru.io/volume-name" ) var log = logf.Log.WithName("controller_rpaasinstance") @@ -195,6 +199,10 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile } if plan.Spec.Config.CacheHeaterEnabled { + err = r.reconcileCacheHeaterCronJob(instance, plan) + if err != nil { + return reconcile.Result{}, err + } err = r.reconcileCacheHeaterVolume(instance, plan) if err != nil { return reconcile.Result{}, err @@ -426,6 +434,84 @@ func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) erro return err } +func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { + cronName := instance.Name + "-heater-cron-job" + cronJob := &batchv1beta1.CronJob{} + + err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) + isNotFound := k8sErrors.IsNotFound(err) + if err != nil && !isNotFound { + return err + } else if !isNotFound { + logrus.Infof("CronJob %s is found, skipping creation", cronName) + return nil + } + + podCommand := "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" + + schedule := defaultCacheHeaterSchedule + if plan.Spec.Config.CacheHeaterSync.Schedule != "" { + schedule = plan.Spec.Config.CacheHeaterSync.Schedule + } + + image := defaultCacheHeaterCronImage + if plan.Spec.Config.CacheHeaterSync.Image != "" { + image = plan.Spec.Config.CacheHeaterSync.Image + } + + cronJob = &batchv1beta1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: cronName, + Namespace: instance.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(instance, schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "RpaasInstance", + }), + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1beta1", + Kind: "CronJob", + }, + Spec: batchv1beta1.CronJobSpec{ + Schedule: schedule, + JobTemplate: batchv1beta1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: "rpaas-cache-heater-cronjob", + Containers: []corev1.Container{ + { + Name: "cache-synchronize", + Image: image, + Command: []string{ + "/bin/bash", + }, + Args: []string{ + "-c", + fmt.Sprintf("pods=($(kubectl -n rpaasv2-be-cme get pod -l rpaas.extensions.tsuru.io/service-name=%q -l rpaas.extensions.tsuru.io/instance-name=%q --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); ", instance.Namespace, instance.Name) + + "for pod in ${pods[@]}; do " + + fmt.Sprintf("kubectl -n %q exec ${pod} -- %q;", instance.Namespace, podCommand) + + "if [[ $? == 0 ]]; then exit 0; fi; " + + "done", + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + }, + }, + } + err = r.client.Create(context.TODO(), cronJob) + if err != nil { + return err + } + return nil +} func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { pvcName := instance.Name + "-heater-volume" @@ -444,13 +530,14 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R cacheHeaterStorage = plan.Spec.Config.CacheHeaterStorage } volumeMode := corev1.PersistentVolumeFilesystem - labels := map[string]string{} - if teamOwner := instance.TeamOwner(); teamOwner != "" { - labels[volumeTeamLabel] = teamOwner - } - for k, v := range cacheHeaterStorage.VolumeLabels { - labels[k] = v - } + labels := map[string]string{} + if teamOwner := instance.TeamOwner(); teamOwner != "" { + labels[volumeTeamLabel] = teamOwner + } + for k, v := range cacheHeaterStorage.VolumeLabels { + labels[k] = v + } + pvc = &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 62b77ab90..f1cc9593f 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -15,6 +15,8 @@ import ( "github.com/tsuru/rpaas-operator/pkg/apis" "github.com/tsuru/rpaas-operator/pkg/apis/extensions/v1alpha1" autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + batchv1 "k8s.io/api/batch/v1" + batchv1beta1 "k8s.io/api/batch/v1beta1" corev1 "k8s.io/api/core/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -736,7 +738,7 @@ func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" - instance1.SetTeamOwner("team-one") + instance1.SetTeamOwner("team-one") resources := []runtime.Object{} scheme := newScheme() @@ -778,7 +780,7 @@ func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { func Test_reconcileHeaterVolumeLabels(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" - instance1.SetTeamOwner("team-one") + instance1.SetTeamOwner("team-one") resources := []runtime.Object{} scheme := newScheme() @@ -796,8 +798,8 @@ func Test_reconcileHeaterVolumeLabels(t *testing.T) { CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), VolumeLabels: map[string]string{ - "some-label": "foo", - "other-label": "bar", + "some-label": "foo", + "other-label": "bar", volumeTeamLabel: "another-team", }, }, @@ -1478,12 +1480,18 @@ func TestReconcile(t *testing.T) { CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, + CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ + Schedule: "1 * * * *", + Image: "test/test:latest", + }, }, }, } resources := []runtime.Object{rpaas, plan} scheme := newScheme() corev1.AddToScheme(scheme) + batchv1.AddToScheme(scheme) + batchv1beta1.AddToScheme(scheme) client := fake.NewFakeClientWithScheme(scheme, resources...) reconciler := &ReconcileRpaasInstance{ client: client, @@ -1502,4 +1510,18 @@ func TestReconcile(t *testing.T) { assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instance-heater-volume"}) assert.Equal(t, nginx.Spec.PodTemplate.VolumeMounts[0].Name, "cache-heater-volume") assert.Equal(t, nginx.Spec.PodTemplate.VolumeMounts[0].MountPath, "/var/cache/cache-heater") + + cronJob := &batchv1beta1.CronJob{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "my-instance-heater-cron-job", Namespace: rpaas.Namespace}, cronJob) + require.NoError(t, err) + + assert.Equal(t, "1 * * * *", cronJob.Spec.Schedule) + templateSpec := cronJob.Spec.JobTemplate.Spec.Template.Spec + assert.Equal(t, "test/test:latest", templateSpec.Containers[0].Image) + assert.Equal(t, "pods=($(kubectl -n rpaasv2-be-cme get pod -l rpaas.extensions.tsuru.io/service-name=\"default\" -l rpaas.extensions.tsuru.io/instance-name=\"my-instance\" --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}'));"+ + " for pod in ${pods[@]}; "+ + "do kubectl -n \"default\" exec ${pod} -- \"rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater\";"+ + "if [[ $? == 0 ]]; then exit 0; fi; done", + templateSpec.Containers[0].Args[1], + ) } From d9911f68e4de6b45be1eb87b2466e1c99fa533a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 17 Apr 2020 14:15:20 -0300 Subject: [PATCH 21/47] Add possibility to customize cronjob command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Wilson Júnior --- .../extensions/v1alpha1/rpaasplan_types.go | 3 + .../rpaasinstance/rpaasinstance_controller.go | 77 ++++++++++++++----- .../rpaasinstance_controller_test.go | 54 ++++++++++--- 3 files changed, 106 insertions(+), 28 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 599a6811b..404d16cac 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -92,6 +92,9 @@ type CacheHeaterSyncSpec struct { // Container image used to sync the containers // default is bitnami/kubectl:latest Image string `json:"image,omitempty"` + + // Cmds is used to customize command used to sync memory cache to persistent storage + Cmds []string `json:"cmds,omitempty"` } type CacheHeaterStorage struct { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 333af6e3f..00324aac5 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -44,14 +44,28 @@ import ( ) const ( - defaultConfigHistoryLimit = 10 - defaultCacheHeaterCronImage = "bitnami/kubectl:latest" - defaultCacheHeaterSchedule = "* * * * *" - + defaultConfigHistoryLimit = 10 + defaultCacheHeaterCronImage = "bitnami/kubectl:latest" + defaultCacheHeaterSchedule = "* * * * *" defaultPortAllocationResource = "default" volumeTeamLabel = "tsuru.io/volume-name" ) +var ( + defaultCacheHeaterCmds = []string{ + "/bin/bash", + "-c", + ` +pods=($(kubectl -n rpaasv2-be-cme get pod -l rpaas.extensions.tsuru.io/service-name=${SERVICE_NAME} -l rpaas.extensions.tsuru.io/instance-name=${INSTANCE_NAME} --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); +for pod in ${pods[@]}; do + kubectl -n ${SERVICE_NAME} exec ${pod} -- ${POD_CMD}; + if [[ $? == 0 ]]; then + exit 0; + fi +done +`} +) + var log = logf.Log.WithName("controller_rpaasinstance") // Add creates a new RpaasInstance Controller and adds it to the Manager. The Manager will set fields on the Controller @@ -100,6 +114,11 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { OwnerType: &extensionsv1alpha1.RpaasInstance{}, }) + err = c.Watch(&source.Kind{Type: &batchv1beta1.CronJob{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &extensionsv1alpha1.RpaasInstance{}, + }) + return err } @@ -208,7 +227,11 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile return reconcile.Result{}, err } } else { - err = r.destroyCacheHeaterVolume(instance, plan.Spec.Config.CacheHeaterStorage) + err = r.destroyCacheHeaterCronJob(instance) + if err != nil { + return reconcile.Result{}, err + } + err = r.destroyCacheHeaterVolume(instance) if err != nil { return reconcile.Result{}, err } @@ -439,12 +462,11 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1. cronJob := &batchv1beta1.CronJob{} err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) - isNotFound := k8sErrors.IsNotFound(err) - if err != nil && !isNotFound { - return err - } else if !isNotFound { + if err == nil { logrus.Infof("CronJob %s is found, skipping creation", cronName) return nil + } else if !k8sErrors.IsNotFound(err) { + return err } podCommand := "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" @@ -459,6 +481,11 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1. image = plan.Spec.Config.CacheHeaterSync.Image } + cmds := defaultCacheHeaterCmds + if len(plan.Spec.Config.CacheHeaterSync.Cmds) > 0 { + cmds = plan.Spec.Config.CacheHeaterSync.Cmds + } + cronJob = &batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ Name: cronName, @@ -487,15 +514,13 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1. Name: "cache-synchronize", Image: image, Command: []string{ - "/bin/bash", + cmds[0], }, - Args: []string{ - "-c", - fmt.Sprintf("pods=($(kubectl -n rpaasv2-be-cme get pod -l rpaas.extensions.tsuru.io/service-name=%q -l rpaas.extensions.tsuru.io/instance-name=%q --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); ", instance.Namespace, instance.Name) + - "for pod in ${pods[@]}; do " + - fmt.Sprintf("kubectl -n %q exec ${pod} -- %q;", instance.Namespace, podCommand) + - "if [[ $? == 0 ]]; then exit 0; fi; " + - "done", + Args: cmds[1:], + Env: []corev1.EnvVar{ + {Name: "SERVICE_NAME", Value: instance.Namespace}, + {Name: "INSTANCE_NAME", Value: instance.Name}, + {Name: "POD_CMD", Value: podCommand}, }, }, }, @@ -512,6 +537,22 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1. } return nil } + +func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(instance *v1alpha1.RpaasInstance) error { + cronName := instance.Name + "-heater-cron-job" + cronJob := &batchv1beta1.CronJob{} + + err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) + isNotFound := k8sErrors.IsNotFound(err) + if err != nil && !isNotFound { + return err + } else if isNotFound { + logrus.Infof("CronJob %s is not found, skipping destruction", cronName) + return nil + } + + return r.client.Delete(context.TODO(), cronJob) +} func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { pvcName := instance.Name + "-heater-volume" @@ -589,7 +630,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R return nil } -func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance, storageConfig *v1alpha1.CacheHeaterStorage) error { +func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { pvcName := instance.Name + "-heater-volume" pvc := &corev1.PersistentVolumeClaim{} diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index f1cc9593f..2b6829ee3 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -905,9 +905,6 @@ func Test_destroyHeaterVolume(t *testing.T) { Namespace: "default", }, } - storageConfig := &v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), - } resources := []runtime.Object{pvc} scheme := newScheme() corev1.AddToScheme(scheme) @@ -918,7 +915,7 @@ func Test_destroyHeaterVolume(t *testing.T) { scheme: newScheme(), } - err := reconciler.destroyCacheHeaterVolume(instance1, storageConfig) + err := reconciler.destroyCacheHeaterVolume(instance1) require.NoError(t, err) pvc = &corev1.PersistentVolumeClaim{} @@ -926,6 +923,36 @@ func Test_destroyHeaterVolume(t *testing.T) { require.True(t, k8sErrors.IsNotFound(err)) } +func Test_destroyHeaterCronJob(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + cronJob := &batchv1beta1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance1.Name + "-heater-cron-job", + Namespace: instance1.Namespace, + }, + } + + resources := []runtime.Object{cronJob} + scheme := newScheme() + corev1.AddToScheme(scheme) + batchv1beta1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + err := reconciler.destroyCacheHeaterCronJob(instance1) + require.NoError(t, err) + + cronJob = &batchv1beta1.CronJob{} + + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-cron-job", Namespace: instance1.Namespace}, cronJob) + require.True(t, k8sErrors.IsNotFound(err)) +} func int32Ptr(n int32) *int32 { return &n } @@ -1483,6 +1510,11 @@ func TestReconcile(t *testing.T) { CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ Schedule: "1 * * * *", Image: "test/test:latest", + Cmds: []string{ + "/bin/bash", + "-c", + "echo 'this is a test'", + }, }, }, }, @@ -1518,10 +1550,12 @@ func TestReconcile(t *testing.T) { assert.Equal(t, "1 * * * *", cronJob.Spec.Schedule) templateSpec := cronJob.Spec.JobTemplate.Spec.Template.Spec assert.Equal(t, "test/test:latest", templateSpec.Containers[0].Image) - assert.Equal(t, "pods=($(kubectl -n rpaasv2-be-cme get pod -l rpaas.extensions.tsuru.io/service-name=\"default\" -l rpaas.extensions.tsuru.io/instance-name=\"my-instance\" --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}'));"+ - " for pod in ${pods[@]}; "+ - "do kubectl -n \"default\" exec ${pod} -- \"rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater\";"+ - "if [[ $? == 0 ]]; then exit 0; fi; done", - templateSpec.Containers[0].Args[1], - ) + assert.Equal(t, "/bin/bash", templateSpec.Containers[0].Command[0]) + assert.Equal(t, "-c", templateSpec.Containers[0].Args[0]) + assert.Equal(t, "echo 'this is a test'", templateSpec.Containers[0].Args[1]) + assert.Equal(t, []corev1.EnvVar{ + {Name: "SERVICE_NAME", Value: "default"}, + {Name: "INSTANCE_NAME", Value: "my-instance"}, + {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater"}, + }, templateSpec.Containers[0].Env) } From 6df2d311271f7e903765d8608ded86da112c1e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 17 Apr 2020 14:40:42 -0300 Subject: [PATCH 22/47] Avoid struct pointers in CRDs, looking at kubernetes API the major CRDs avoid struct pointer --- Makefile | 3 +- .../extensions/v1alpha1/rpaasplan_types.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 35 ++++++++++++++++--- .../rpaasinstance/rpaasinstance_controller.go | 11 ++---- .../rpaasinstance_controller_test.go | 14 ++++---- 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 45ccf74f7..feb2bf6f8 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ KUBECONFIG ?= ~/.kube/config git_tag := $(shell git describe --tags --abbrev=0 2>/dev/null || echo 'untagged') git_commit := $(shell git rev-parse HEAD 2>/dev/null | cut -c1-7) +go_root := $(shell go env GOROOT) RPAAS_OPERATOR_VERSION ?= $(git_tag)/$(git_commit) GO_LDFLAGS ?= -X=github.com/tsuru/rpaas-operator/version.Version=$(RPAAS_OPERATOR_VERSION) @@ -34,7 +35,7 @@ local: deploy/crds operator-sdk up local --go-ldflags $(GO_LDFLAGS) generate: - operator-sdk generate k8s + GOROOT=$(go_root) operator-sdk generate k8s build: build/plugin/rpaasv2 operator-sdk build $(IMAGE_OPERATOR):$(TAG) --go-build-args "-ldflags $(GO_LDFLAGS)" diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 404d16cac..1fa03e450 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -67,7 +67,7 @@ type NginxConfig struct { CacheZoneSize string `json:"cacheZoneSize,omitempty"` CacheHeaterEnabled bool `json:"cacheHeaterEnabled,omitempty"` - CacheHeaterStorage *CacheHeaterStorage `json:"cacheHeaterStorage,omitempty"` + CacheHeaterStorage CacheHeaterStorage `json:"cacheHeaterStorage,omitempty"` CacheHeaterSync CacheHeaterSyncSpec `json:"cacheHeaterSync,omitempty"` HTTPListenOptions string `json:"httpListenOptions,omitempty"` diff --git a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go index adca8a43c..6be8f9e09 100644 --- a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go @@ -55,6 +55,13 @@ func (in *CacheHeaterStorage) DeepCopyInto(out *CacheHeaterStorage) { *out = new(string) **out = **in } + if in.VolumeLabels != nil { + in, out := &in.VolumeLabels, &out.VolumeLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } @@ -68,6 +75,27 @@ func (in *CacheHeaterStorage) DeepCopy() *CacheHeaterStorage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheHeaterSyncSpec) DeepCopyInto(out *CacheHeaterSyncSpec) { + *out = *in + if in.Cmds != nil { + in, out := &in.Cmds, &out.Cmds + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheHeaterSyncSpec. +func (in *CacheHeaterSyncSpec) DeepCopy() *CacheHeaterSyncSpec { + if in == nil { + return nil + } + out := new(CacheHeaterSyncSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Location) DeepCopyInto(out *Location) { *out = *in @@ -113,11 +141,8 @@ func (in *NginxConfig) DeepCopyInto(out *NginxConfig) { *out = new(bool) **out = **in } - if in.CacheHeaterStorage != nil { - in, out := &in.CacheHeaterStorage, &out.CacheHeaterStorage - *out = new(CacheHeaterStorage) - (*in).DeepCopyInto(*out) - } + in.CacheHeaterStorage.DeepCopyInto(&out.CacheHeaterStorage) + in.CacheHeaterSync.DeepCopyInto(&out.CacheHeaterSync) if in.VTSEnabled != nil { in, out := &in.VTSEnabled, &out.VTSEnabled *out = new(bool) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 00324aac5..0c83ef044 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -566,10 +566,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R return nil } - cacheHeaterStorage := &v1alpha1.CacheHeaterStorage{} - if plan.Spec.Config.CacheHeaterStorage != nil { - cacheHeaterStorage = plan.Spec.Config.CacheHeaterStorage - } + cacheHeaterStorage := plan.Spec.Config.CacheHeaterStorage volumeMode := corev1.PersistentVolumeFilesystem labels := map[string]string{} if teamOwner := instance.TeamOwner(); teamOwner != "" { @@ -606,10 +603,8 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R } storageSize := plan.Spec.Config.CacheSize - if cacheHeaterStorage := plan.Spec.Config.CacheHeaterStorage; cacheHeaterStorage != nil { - if cacheHeaterStorage.StorageSize != "" { - storageSize = cacheHeaterStorage.StorageSize - } + if cacheHeaterStorage.StorageSize != "" { + storageSize = cacheHeaterStorage.StorageSize } if storageSize != "" { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 2b6829ee3..a13a94867 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -675,7 +675,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, }, @@ -712,7 +712,7 @@ func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), VolumeLabels: map[string]string{ "some-label": "foo", @@ -753,7 +753,7 @@ func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), VolumeLabels: map[string]string{ "some-label": "foo", @@ -795,7 +795,7 @@ func Test_reconcileHeaterVolumeLabels(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), VolumeLabels: map[string]string{ "some-label": "foo", @@ -838,7 +838,7 @@ func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ CacheSize: "10Gi", - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, }, @@ -875,7 +875,7 @@ func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ CacheSize: "10Gi", - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), StorageSize: "100Gi", }, @@ -1504,7 +1504,7 @@ func TestReconcile(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ CacheHeaterEnabled: true, - CacheHeaterStorage: &v1alpha1.CacheHeaterStorage{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ From f8212706af835ce6b7cddf4b49d44e7f31f21281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 20 Apr 2020 13:59:51 -0300 Subject: [PATCH 23/47] Add support to update cronjob when plan changes --- .../rpaasinstance/rpaasinstance_controller.go | 161 ++++++++++-------- .../rpaasinstance_controller_test.go | 76 +++++++++ 2 files changed, 163 insertions(+), 74 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 0c83ef044..45e133a10 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -49,6 +49,9 @@ const ( defaultCacheHeaterSchedule = "* * * * *" defaultPortAllocationResource = "default" volumeTeamLabel = "tsuru.io/volume-name" + + cacheHeaterCronJobSuffix = "-heater-cron-job" + cacheHeaterVolumeSuffix = "-heater-volume" ) var ( @@ -458,88 +461,27 @@ func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) erro } func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { - cronName := instance.Name + "-heater-cron-job" - cronJob := &batchv1beta1.CronJob{} - - err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) - if err == nil { - logrus.Infof("CronJob %s is found, skipping creation", cronName) - return nil - } else if !k8sErrors.IsNotFound(err) { + foundCronJob := &batchv1beta1.CronJob{} + cronName := instance.Name + cacheHeaterCronJobSuffix + err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, foundCronJob) + if err != nil && !k8sErrors.IsNotFound(err) { return err } - podCommand := "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" - - schedule := defaultCacheHeaterSchedule - if plan.Spec.Config.CacheHeaterSync.Schedule != "" { - schedule = plan.Spec.Config.CacheHeaterSync.Schedule - } - - image := defaultCacheHeaterCronImage - if plan.Spec.Config.CacheHeaterSync.Image != "" { - image = plan.Spec.Config.CacheHeaterSync.Image + newestCronJob := newCronJob(instance, plan) + if k8sErrors.IsNotFound(err) { + return r.client.Create(context.TODO(), newestCronJob) } - cmds := defaultCacheHeaterCmds - if len(plan.Spec.Config.CacheHeaterSync.Cmds) > 0 { - cmds = plan.Spec.Config.CacheHeaterSync.Cmds + if !reflect.DeepEqual(foundCronJob.Spec, newestCronJob.Spec) { + return r.client.Update(context.TODO(), newestCronJob) } - cronJob = &batchv1beta1.CronJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: cronName, - Namespace: instance.Namespace, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(instance, schema.GroupVersionKind{ - Group: v1alpha1.SchemeGroupVersion.Group, - Version: v1alpha1.SchemeGroupVersion.Version, - Kind: "RpaasInstance", - }), - }, - }, - TypeMeta: metav1.TypeMeta{ - APIVersion: "batch/v1beta1", - Kind: "CronJob", - }, - Spec: batchv1beta1.CronJobSpec{ - Schedule: schedule, - JobTemplate: batchv1beta1.JobTemplateSpec{ - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - ServiceAccountName: "rpaas-cache-heater-cronjob", - Containers: []corev1.Container{ - { - Name: "cache-synchronize", - Image: image, - Command: []string{ - cmds[0], - }, - Args: cmds[1:], - Env: []corev1.EnvVar{ - {Name: "SERVICE_NAME", Value: instance.Namespace}, - {Name: "INSTANCE_NAME", Value: instance.Name}, - {Name: "POD_CMD", Value: podCommand}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - }, - }, - } - err = r.client.Create(context.TODO(), cronJob) - if err != nil { - return err - } return nil } func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(instance *v1alpha1.RpaasInstance) error { - cronName := instance.Name + "-heater-cron-job" + cronName := instance.Name + cacheHeaterCronJobSuffix cronJob := &batchv1beta1.CronJob{} err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) @@ -554,7 +496,7 @@ func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(instance *v1alpha1.Rp return r.client.Delete(context.TODO(), cronJob) } func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { - pvcName := instance.Name + "-heater-volume" + pvcName := instance.Name + cacheHeaterVolumeSuffix pvc := &corev1.PersistentVolumeClaim{} err := r.client.Get(context.TODO(), types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) @@ -626,7 +568,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R } func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { - pvcName := instance.Name + "-heater-volume" + pvcName := instance.Name + cacheHeaterVolumeSuffix pvc := &corev1.PersistentVolumeClaim{} err := r.client.Get(context.TODO(), types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) @@ -813,7 +755,7 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config Name: "cache-heater-volume", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Name + "-heater-volume", + ClaimName: instance.Name + cacheHeaterVolumeSuffix, }, }, }) @@ -890,6 +832,77 @@ func newHPA(instance v1alpha1.RpaasInstance, nginx nginxv1alpha1.Nginx) autoscal } } +func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *batchv1beta1.CronJob { + cronName := instance.Name + cacheHeaterCronJobSuffix + podCommand := "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" + + schedule := defaultCacheHeaterSchedule + if plan.Spec.Config.CacheHeaterSync.Schedule != "" { + schedule = plan.Spec.Config.CacheHeaterSync.Schedule + } + + image := defaultCacheHeaterCronImage + if plan.Spec.Config.CacheHeaterSync.Image != "" { + image = plan.Spec.Config.CacheHeaterSync.Image + } + + cmds := defaultCacheHeaterCmds + if len(plan.Spec.Config.CacheHeaterSync.Cmds) > 0 { + cmds = plan.Spec.Config.CacheHeaterSync.Cmds + } + return &batchv1beta1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: cronName, + Namespace: instance.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(instance, schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "RpaasInstance", + }), + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1beta1", + Kind: "CronJob", + }, + Spec: batchv1beta1.CronJobSpec{ + Schedule: schedule, + ConcurrencyPolicy: batchv1beta1.ForbidConcurrent, + JobTemplate: batchv1beta1.JobTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "k8s-app": cronName, + }, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: "rpaas-cache-heater-cronjob", + Containers: []corev1.Container{ + { + Name: "cache-synchronize", + Image: image, + Command: []string{ + cmds[0], + }, + Args: cmds[1:], + Env: []corev1.EnvVar{ + {Name: "SERVICE_NAME", Value: instance.Namespace}, + {Name: "INSTANCE_NAME", Value: instance.Name}, + {Name: "POD_CMD", Value: podCommand}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + }, + }, + } +} + func shouldDeleteOldConfig(instance *v1alpha1.RpaasInstance, configList *corev1.ConfigMapList) bool { limit := defaultConfigHistoryLimit diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index a13a94867..ed4873f05 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -923,6 +923,82 @@ func Test_destroyHeaterVolume(t *testing.T) { require.True(t, k8sErrors.IsNotFound(err)) } +func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) + batchv1beta1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{}, + } + + err := reconciler.reconcileCacheHeaterCronJob(instance1, plan) + require.NoError(t, err) + + cronJob := &batchv1beta1.CronJob{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-cron-job", Namespace: instance1.Namespace}, cronJob) + require.NoError(t, err) + + assert.Equal(t, "RpaasInstance", cronJob.ObjectMeta.OwnerReferences[0].Kind) + assert.Equal(t, instance1.Name, cronJob.ObjectMeta.OwnerReferences[0].Name) +} + +func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { + instance1 := newEmptyRpaasInstance() + instance1.Name = "instance-1" + + previousCronJob := &batchv1beta1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance1.Name + "-heater-cronjob", + }, + Spec: batchv1beta1.CronJobSpec{ + Schedule: "old-schedule", + }, + } + + resources := []runtime.Object{previousCronJob} + scheme := newScheme() + corev1.AddToScheme(scheme) + batchv1beta1.AddToScheme(scheme) + + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + + plan := &v1alpha1.RpaasPlan{ + Spec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ + Schedule: "new-schedule", + }, + }, + }, + } + + err := reconciler.reconcileCacheHeaterCronJob(instance1, plan) + require.NoError(t, err) + + cronJob := &batchv1beta1.CronJob{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-cron-job", Namespace: instance1.Namespace}, cronJob) + require.NoError(t, err) + + assert.Equal(t, "RpaasInstance", cronJob.ObjectMeta.OwnerReferences[0].Kind) + assert.Equal(t, instance1.Name, cronJob.ObjectMeta.OwnerReferences[0].Name) + assert.Equal(t, "new-schedule", cronJob.Spec.Schedule) +} + func Test_destroyHeaterCronJob(t *testing.T) { instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" From 27232ebb9a5a5c30840eb0d5555617fca9f0c607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 20 Apr 2020 16:31:44 -0300 Subject: [PATCH 24/47] Fix mistake of hardcoded namespace --- .../rpaasinstance/rpaasinstance_controller.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 45e133a10..5da313269 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -59,7 +59,7 @@ var ( "/bin/bash", "-c", ` -pods=($(kubectl -n rpaasv2-be-cme get pod -l rpaas.extensions.tsuru.io/service-name=${SERVICE_NAME} -l rpaas.extensions.tsuru.io/instance-name=${INSTANCE_NAME} --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); +pods=($(kubectl -n ${SERVICE_NAME} get pod -l rpaas.extensions.tsuru.io/service-name=${SERVICE_NAME} -l rpaas.extensions.tsuru.io/instance-name=${INSTANCE_NAME} --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); for pod in ${pods[@]}; do kubectl -n ${SERVICE_NAME} exec ${pod} -- ${POD_CMD}; if [[ $? == 0 ]]; then @@ -870,18 +870,19 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat Schedule: schedule, ConcurrencyPolicy: batchv1beta1.ForbidConcurrent, JobTemplate: batchv1beta1.JobTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": cronName, - }, - }, + Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "k8s-app": cronName, + }, + }, Spec: corev1.PodSpec{ ServiceAccountName: "rpaas-cache-heater-cronjob", Containers: []corev1.Container{ { - Name: "cache-synchronize", + Name: cronName, Image: image, Command: []string{ cmds[0], From 0422dac0ede699a1cc0c8e3ea7dc895a126537b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 22 Apr 2020 10:09:00 -0300 Subject: [PATCH 25/47] Avoid longer container-name use tags to describe how log will work --- .../rpaasinstance/rpaasinstance_controller.go | 5 +++-- .../rpaasinstance_controller_test.go | 15 +++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 5da313269..1cb68e02e 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -875,14 +875,15 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "k8s-app": cronName, + "log-app-name": instance.Name, + "log-process-name": "cache-synchronize", }, }, Spec: corev1.PodSpec{ ServiceAccountName: "rpaas-cache-heater-cronjob", Containers: []corev1.Container{ { - Name: cronName, + Name: "cache-synchronize", Image: image, Command: []string{ cmds[0], diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index ed4873f05..0ea17720e 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -1624,14 +1624,17 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) assert.Equal(t, "1 * * * *", cronJob.Spec.Schedule) - templateSpec := cronJob.Spec.JobTemplate.Spec.Template.Spec - assert.Equal(t, "test/test:latest", templateSpec.Containers[0].Image) - assert.Equal(t, "/bin/bash", templateSpec.Containers[0].Command[0]) - assert.Equal(t, "-c", templateSpec.Containers[0].Args[0]) - assert.Equal(t, "echo 'this is a test'", templateSpec.Containers[0].Args[1]) + podTemplateSpec := cronJob.Spec.JobTemplate.Spec.Template + podSpec := podTemplateSpec.Spec + assert.Equal(t, "test/test:latest", podSpec.Containers[0].Image) + assert.Equal(t, "/bin/bash", podSpec.Containers[0].Command[0]) + assert.Equal(t, "-c", podSpec.Containers[0].Args[0]) + assert.Equal(t, "echo 'this is a test'", podSpec.Containers[0].Args[1]) + assert.Equal(t, "my-instance", podTemplateSpec.ObjectMeta.Labels["log-app-name"]) + assert.Equal(t, "cache-synchronize", podTemplateSpec.ObjectMeta.Labels["log-process-name"]) assert.Equal(t, []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater"}, - }, templateSpec.Containers[0].Env) + }, podSpec.Containers[0].Env) } From 5a54a7a3874603bdbc0179943165821411d4460b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 22 Apr 2020 15:25:33 -0300 Subject: [PATCH 26/47] Use resource.Quantity instead of string for volume sizes --- .../pkg/rpaas/nginx/configuration_render.go | 19 ++++++++++-- .../rpaas/nginx/configuration_render_test.go | 14 ++++++--- .../extensions/v1alpha1/rpaasplan_types.go | 19 ++++++------ .../rpaasinstance/rpaasinstance_controller.go | 17 +++------- .../rpaasinstance_controller_test.go | 31 ++++++++++++------- 5 files changed, 61 insertions(+), 39 deletions(-) diff --git a/internal/pkg/rpaas/nginx/configuration_render.go b/internal/pkg/rpaas/nginx/configuration_render.go index 6aadf53b9..c03e52846 100644 --- a/internal/pkg/rpaas/nginx/configuration_render.go +++ b/internal/pkg/rpaas/nginx/configuration_render.go @@ -12,6 +12,7 @@ import ( "github.com/tsuru/rpaas-operator/pkg/apis/extensions/v1alpha1" "github.com/tsuru/rpaas-operator/pkg/util" + "k8s.io/apimachinery/pkg/api/resource" ) type ConfigurationRenderer interface { @@ -145,6 +146,19 @@ func managePort(instance *v1alpha1.RpaasInstance) int32 { return defaultManagePort } +func k8sQuantityToNginx(quantity *resource.Quantity) string { + if quantity == nil || quantity.IsZero() { + return "0" + } + repr := quantity.String() + + if repr[len(repr)-1] == 'i' { + return repr[0 : len(repr)-1] + + } + return repr +} + var templateFuncs = template.FuncMap(map[string]interface{}{ "boolValue": v1alpha1.BoolValue, "buildLocationKey": buildLocationKey, @@ -159,6 +173,7 @@ var templateFuncs = template.FuncMap(map[string]interface{}{ "contains": strings.Contains, "hasPrefix": strings.HasPrefix, "hasSuffix": strings.HasSuffix, + "k8sQuantityToNginx": k8sQuantityToNginx, }) var defaultMainTemplate = template.Must(template.New("main"). @@ -217,9 +232,9 @@ http { proxy_http_version 1.1; {{- if boolValue $config.CacheEnabled }} - proxy_cache_path {{ $config.CachePath }}/nginx levels=1:2 keys_zone=rpaas:{{ $config.CacheZoneSize }} + proxy_cache_path {{ $config.CachePath }}/nginx levels=1:2 keys_zone=rpaas:{{ k8sQuantityToNginx $config.CacheZoneSize }} {{- with $config.CacheInactive }} inactive={{ . }}{{ end }} - {{- with $config.CacheSize }} max_size={{ . }}{{ end }} + {{- with $config.CacheSize }} max_size={{ k8sQuantityToNginx . }}{{ end }} {{- with $config.CacheLoaderFiles }} loader_files={{ . }}{{ end }}; proxy_temp_path {{ $config.CachePath }}/nginx_tmp 1 2; diff --git a/internal/pkg/rpaas/nginx/configuration_render_test.go b/internal/pkg/rpaas/nginx/configuration_render_test.go index 8108c34fc..5d547b190 100644 --- a/internal/pkg/rpaas/nginx/configuration_render_test.go +++ b/internal/pkg/rpaas/nginx/configuration_render_test.go @@ -12,9 +12,13 @@ import ( nginxv1alpha1 "github.com/tsuru/nginx-operator/pkg/apis/nginx/v1alpha1" "github.com/tsuru/rpaas-operator/pkg/apis/extensions/v1alpha1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) func TestRpaasConfigurationRenderer_Render(t *testing.T) { + size100MB := resource.MustParse("100Mi") + size300MB := resource.MustParse("300Mi") + tests := []struct { name string blocks ConfigurationBlocks @@ -70,12 +74,12 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { Config: &v1alpha1.NginxConfig{ CacheEnabled: v1alpha1.Bool(true), CachePath: "/path/to/cache/dir", - CacheZoneSize: "100m", + CacheZoneSize: &size100MB, }, Instance: &v1alpha1.RpaasInstance{}, }, assertion: func(t *testing.T, result string) { - assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100m;`, result) + assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100M;`, result) assert.Regexp(t, `proxy_temp_path /path/to/cache/dir/nginx_tmp 1 2;`, result) assert.Regexp(t, `server { \s+listen 8800; @@ -95,13 +99,13 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { CachePath: "/path/to/cache/dir", CacheInactive: "12h", CacheLoaderFiles: 1000, - CacheSize: "300m", - CacheZoneSize: "100m", + CacheSize: &size300MB, + CacheZoneSize: &size100MB, }, Instance: &v1alpha1.RpaasInstance{}, }, assertion: func(t *testing.T, result string) { - assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100m inactive=12h max_size=300m loader_files=1000;`, result) + assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100M inactive=12h max_size=300M loader_files=1000;`, result) assert.Regexp(t, `proxy_temp_path /path/to/cache/dir/nginx_tmp 1 2;`, result) assert.Regexp(t, `server { \s+listen 8800; diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 1fa03e450..676a6059e 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -6,6 +6,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,12 +60,12 @@ type NginxConfig struct { UpstreamKeepalive int `json:"upstreamKeepalive,omitempty"` - CacheEnabled *bool `json:"cacheEnabled,omitempty"` - CacheInactive string `json:"cacheInactive,omitempty"` - CacheLoaderFiles int `json:"cacheLoaderFiles,omitempty"` - CachePath string `json:"cachePath,omitempty"` - CacheSize string `json:"cacheSize,omitempty"` - CacheZoneSize string `json:"cacheZoneSize,omitempty"` + CacheEnabled *bool `json:"cacheEnabled,omitempty"` + CacheInactive string `json:"cacheInactive,omitempty"` + CacheLoaderFiles int `json:"cacheLoaderFiles,omitempty"` + CachePath string `json:"cachePath,omitempty"` + CacheSize *resource.Quantity `json:"cacheSize,omitempty"` + CacheZoneSize *resource.Quantity `json:"cacheZoneSize,omitempty"` CacheHeaterEnabled bool `json:"cacheHeaterEnabled,omitempty"` CacheHeaterStorage CacheHeaterStorage `json:"cacheHeaterStorage,omitempty"` @@ -98,9 +99,9 @@ type CacheHeaterSyncSpec struct { } type CacheHeaterStorage struct { - StorageClassName *string `json:"storageClassName,omitempty"` - StorageSize string `json:"storageSize,omitempty"` - VolumeLabels map[string]string `json:"volumeLabels,omitempty"` + StorageClassName *string `json:"storageClassName,omitempty"` + StorageSize *resource.Quantity `json:"storageSize,omitempty"` + VolumeLabels map[string]string `json:"volumeLabels,omitempty"` } func Bool(v bool) *bool { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 1cb68e02e..4ed755f9b 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -28,8 +28,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" k8sErrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - k8sResources "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -545,18 +543,14 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R } storageSize := plan.Spec.Config.CacheSize - if cacheHeaterStorage.StorageSize != "" { + if cacheHeaterStorage.StorageSize != nil && !cacheHeaterStorage.StorageSize.IsZero() { storageSize = cacheHeaterStorage.StorageSize } - if storageSize != "" { - parsedSize, err := resource.ParseQuantity(storageSize) - if err != nil { - return err - } + if storageSize != nil && !storageSize.IsZero() { pvc.Spec.Resources = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - "storage": parsedSize, + "storage": *storageSize, }, } } @@ -711,9 +705,8 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config if v1alpha1.BoolValue(plan.Spec.Config.CacheEnabled) { cacheConfig.Path = plan.Spec.Config.CachePath cacheConfig.InMemory = true - cacheMaxSize, err := k8sResources.ParseQuantity(plan.Spec.Config.CacheSize) - if err == nil && !cacheMaxSize.IsZero() { - cacheConfig.Size = &cacheMaxSize + if plan.Spec.Config.CacheSize != nil && !plan.Spec.Config.CacheSize.IsZero() { + cacheConfig.Size = plan.Spec.Config.CacheSize } } n := &nginxv1alpha1.Nginx{ diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 0ea17720e..fafb3df69 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -61,7 +61,7 @@ func Test_mergePlans(t *testing.T) { Description: "a", Config: v1alpha1.NginxConfig{ User: "root", - CacheSize: "10", + CacheSize: resourceMustParsePtr("10M"), CacheEnabled: v1alpha1.Bool(true), }, }, @@ -76,7 +76,7 @@ func Test_mergePlans(t *testing.T) { Description: "a", Config: v1alpha1.NginxConfig{ User: "ubuntu", - CacheSize: "10", + CacheSize: resourceMustParsePtr("10M"), CacheEnabled: v1alpha1.Bool(true), }, }, @@ -87,7 +87,7 @@ func Test_mergePlans(t *testing.T) { Description: "a", Config: v1alpha1.NginxConfig{ User: "root", - CacheSize: "10", + CacheSize: resourceMustParsePtr("10M"), CacheEnabled: v1alpha1.Bool(true), }, }, @@ -103,7 +103,7 @@ func Test_mergePlans(t *testing.T) { Description: "a", Config: v1alpha1.NginxConfig{ User: "ubuntu", - CacheSize: "10", + CacheSize: resourceMustParsePtr("10M"), CacheEnabled: v1alpha1.Bool(false), }, }, @@ -837,7 +837,7 @@ func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheSize: "10Gi", + CacheSize: resourceMustParsePtr("10Gi"), CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, @@ -874,10 +874,10 @@ func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheSize: "10Gi", + CacheSize: resourceMustParsePtr("10Gi"), CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), - StorageSize: "100Gi", + StorageSize: resourceMustParsePtr("100Gi"), }, }, }, @@ -1579,6 +1579,8 @@ func TestReconcile(t *testing.T) { }, Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ + CacheEnabled: v1alpha1.Bool(true), + CacheSize: resourceMustParsePtr("100M"), CacheHeaterEnabled: true, CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), @@ -1614,10 +1616,12 @@ func TestReconcile(t *testing.T) { err = client.Get(context.TODO(), types.NamespacedName{Name: rpaas.Name, Namespace: rpaas.Namespace}, nginx) require.NoError(t, err) - assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].Name, "cache-heater-volume") - assert.Equal(t, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instance-heater-volume"}) - assert.Equal(t, nginx.Spec.PodTemplate.VolumeMounts[0].Name, "cache-heater-volume") - assert.Equal(t, nginx.Spec.PodTemplate.VolumeMounts[0].MountPath, "/var/cache/cache-heater") + assert.Equal(t, "cache-heater-volume", nginx.Spec.PodTemplate.Volumes[0].Name) + assert.Equal(t, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instance-heater-volume"}, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim) + assert.Equal(t, "cache-heater-volume", nginx.Spec.PodTemplate.VolumeMounts[0].Name) + assert.Equal(t, "/var/cache/cache-heater", nginx.Spec.PodTemplate.VolumeMounts[0].MountPath) + + assert.Equal(t, resource.MustParse("100M"), *nginx.Spec.Cache.Size) cronJob := &batchv1beta1.CronJob{} err = client.Get(context.TODO(), types.NamespacedName{Name: "my-instance-heater-cron-job", Namespace: rpaas.Namespace}, cronJob) @@ -1638,3 +1642,8 @@ func TestReconcile(t *testing.T) { {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater"}, }, podSpec.Containers[0].Env) } + +func resourceMustParsePtr(fmt string) *resource.Quantity { + qty := resource.MustParse(fmt) + return &qty +} From 1ea023578b80a17f25555bf8a12042aa748b066b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 22 Apr 2020 15:46:24 -0300 Subject: [PATCH 27/47] Use strings.TrimSuffix instead of manual trim --- internal/pkg/rpaas/nginx/configuration_render.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/pkg/rpaas/nginx/configuration_render.go b/internal/pkg/rpaas/nginx/configuration_render.go index c03e52846..4b33ee3c8 100644 --- a/internal/pkg/rpaas/nginx/configuration_render.go +++ b/internal/pkg/rpaas/nginx/configuration_render.go @@ -152,11 +152,7 @@ func k8sQuantityToNginx(quantity *resource.Quantity) string { } repr := quantity.String() - if repr[len(repr)-1] == 'i' { - return repr[0 : len(repr)-1] - - } - return repr + return strings.TrimSuffix(repr, "i") } var templateFuncs = template.FuncMap(map[string]interface{}{ From 8725ddca2c81481a348e478704d0af2d05a55dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 22 Apr 2020 16:55:35 -0300 Subject: [PATCH 28/47] improve k8sQuantityToNginx to handle size accurately --- .../pkg/rpaas/nginx/configuration_render.go | 13 +++++++++-- .../rpaas/nginx/configuration_render_test.go | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/internal/pkg/rpaas/nginx/configuration_render.go b/internal/pkg/rpaas/nginx/configuration_render.go index 4b33ee3c8..9113232f2 100644 --- a/internal/pkg/rpaas/nginx/configuration_render.go +++ b/internal/pkg/rpaas/nginx/configuration_render.go @@ -7,6 +7,7 @@ package nginx import ( "bytes" "fmt" + "strconv" "strings" "text/template" @@ -15,6 +16,8 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) +var terabyte = resource.NewQuantity(1*1024*1024*1024*1024, resource.BinarySI) + type ConfigurationRenderer interface { Render(ConfigurationData) (string, error) } @@ -150,9 +153,15 @@ func k8sQuantityToNginx(quantity *resource.Quantity) string { if quantity == nil || quantity.IsZero() { return "0" } - repr := quantity.String() - return strings.TrimSuffix(repr, "i") + bytesN, _ := quantity.AsInt64() + // nginx is not support terabyte yet + // https://github.com/nginx/nginx/blob/3ba88365b5acef17f01671cd969c909dee5e2cde/src/core/ngx_parse.c#L81-L87 + if quantity.Cmp(*terabyte) >= 0 { + return strconv.Itoa(int(bytesN)) + } + + return strings.TrimSuffix(resource.NewQuantity(bytesN, resource.BinarySI).String(), "i") } var templateFuncs = template.FuncMap(map[string]interface{}{ diff --git a/internal/pkg/rpaas/nginx/configuration_render_test.go b/internal/pkg/rpaas/nginx/configuration_render_test.go index 5d547b190..95b8339fa 100644 --- a/internal/pkg/rpaas/nginx/configuration_render_test.go +++ b/internal/pkg/rpaas/nginx/configuration_render_test.go @@ -562,3 +562,26 @@ func Test_hasRootPath(t *testing.T) { }) } } + +func TestK8sQuantityToNginx(t *testing.T) { + type expectation struct { + k8sQuantity string + nginxQuantity string + } + + expectations := []expectation{ + {"100Ki", "100K"}, + {"100Mi", "100M"}, + {"100M", "100000000"}, + {"1Gi", "1G"}, + {"1G", "1000000000"}, + {"1024Gi", "1099511627776"}, + {"2Ti", "2199023255552"}, + } + + for _, expectation := range expectations { + k8sQuantity := resource.MustParse(expectation.k8sQuantity) + nginxQuantity := k8sQuantityToNginx(&k8sQuantity) + assert.Equal(t, expectation.nginxQuantity, nginxQuantity) + } +} From 5e5fabbd9c8edd2370b718112c2a7a29c310e66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 24 Apr 2020 11:31:44 -0300 Subject: [PATCH 29/47] Add InitContainer to sync cache from persistent volume --- go.mod | 2 +- go.sum | 4 +- .../extensions/v1alpha1/rpaasplan_types.go | 5 +- .../v1alpha1/zz_generated.deepcopy.go | 24 +++++++- .../rpaasinstance/rpaasinstance_controller.go | 55 ++++++++++++++++--- .../rpaasinstance_controller_test.go | 26 ++++++++- 6 files changed, 100 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 4a3f7fca2..d93ec1c48 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.4.0 - github.com/tsuru/nginx-operator v0.5.1 + github.com/tsuru/nginx-operator v0.5.2 github.com/urfave/cli/v2 v2.0.0 github.com/willf/bitset v1.1.10 k8s.io/api v0.0.0 diff --git a/go.sum b/go.sum index 4bb6817f1..024b2e8d8 100644 --- a/go.sum +++ b/go.sum @@ -611,8 +611,8 @@ github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1C github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tsuru/config v0.0.0-20180418191556-87403ee7da02/go.mod h1:bilf/jr5AGl9raMFnhM9hsp+ms6PK752UF89ODoRHW4= -github.com/tsuru/nginx-operator v0.5.1 h1:vgLTB8om6wC91loqZcR5z1v5ONgLuIev1zzxG9WlHfk= -github.com/tsuru/nginx-operator v0.5.1/go.mod h1:hvXj+bM5dUsnOt/PQE1ktHnLxM1ucnhKM0zOdHZyzsg= +github.com/tsuru/nginx-operator v0.5.2 h1:5i+0EKOnvVrkXEiXIxzKT75h4y80dj50kiqZwqlyeUk= +github.com/tsuru/nginx-operator v0.5.2/go.mod h1:hvXj+bM5dUsnOt/PQE1ktHnLxM1ucnhKM0zOdHZyzsg= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index 676a6059e..eefc82f95 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -94,8 +94,9 @@ type CacheHeaterSyncSpec struct { // default is bitnami/kubectl:latest Image string `json:"image,omitempty"` - // Cmds is used to customize command used to sync memory cache to persistent storage - Cmds []string `json:"cmds,omitempty"` + // Cmds that are used to customize command used to sync memory cache to persistent storage + CmdPodToPVC []string `json:"cmdPodToPVC,omitempty"` + CmdPVCToPod []string `json:"cmdPVCToPod,omitempty"` } type CacheHeaterStorage struct { diff --git a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go index 6be8f9e09..7516daa3e 100644 --- a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go @@ -55,6 +55,11 @@ func (in *CacheHeaterStorage) DeepCopyInto(out *CacheHeaterStorage) { *out = new(string) **out = **in } + if in.StorageSize != nil { + in, out := &in.StorageSize, &out.StorageSize + x := (*in).DeepCopy() + *out = &x + } if in.VolumeLabels != nil { in, out := &in.VolumeLabels, &out.VolumeLabels *out = make(map[string]string, len(*in)) @@ -78,8 +83,13 @@ func (in *CacheHeaterStorage) DeepCopy() *CacheHeaterStorage { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheHeaterSyncSpec) DeepCopyInto(out *CacheHeaterSyncSpec) { *out = *in - if in.Cmds != nil { - in, out := &in.Cmds, &out.Cmds + if in.CmdPodToPVC != nil { + in, out := &in.CmdPodToPVC, &out.CmdPodToPVC + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CmdPVCToPod != nil { + in, out := &in.CmdPVCToPod, &out.CmdPVCToPod *out = make([]string, len(*in)) copy(*out, *in) } @@ -141,6 +151,16 @@ func (in *NginxConfig) DeepCopyInto(out *NginxConfig) { *out = new(bool) **out = **in } + if in.CacheSize != nil { + in, out := &in.CacheSize, &out.CacheSize + x := (*in).DeepCopy() + *out = &x + } + if in.CacheZoneSize != nil { + in, out := &in.CacheZoneSize, &out.CacheZoneSize + x := (*in).DeepCopy() + *out = &x + } in.CacheHeaterStorage.DeepCopyInto(&out.CacheHeaterStorage) in.CacheHeaterSync.DeepCopyInto(&out.CacheHeaterSync) if in.VTSEnabled != nil { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 4ed755f9b..973ceb7d0 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -1,4 +1,4 @@ -// Copyright 2019 tsuru authors. All rights reserved. +// Copyright 2020 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -50,10 +50,13 @@ const ( cacheHeaterCronJobSuffix = "-heater-cron-job" cacheHeaterVolumeSuffix = "-heater-volume" + + rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" + rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-heater/nginx /var/cache/nginx/rpaas" ) var ( - defaultCacheHeaterCmds = []string{ + defaultCacheHeaterCmdPodToPVC = []string{ "/bin/bash", "-c", ` @@ -64,6 +67,15 @@ for pod in ${pods[@]}; do exit 0; fi done +`} + + defaultCacheHeaterCmdPVCToPod = []string{ + "/bin/bash", + "-c", + ` +mkdir -p /var/cache/cache-heater/temp; +mkdir -p /var/cache/nginx/rpaas/nginx_tmp; +bash -c ${POD_CMD} `} ) @@ -744,6 +756,11 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config } if plan.Spec.Config.CacheHeaterEnabled { + initCmd := defaultCacheHeaterCmdPVCToPod + if len(plan.Spec.Config.CacheHeaterSync.CmdPVCToPod) > 0 { + initCmd = plan.Spec.Config.CacheHeaterSync.CmdPVCToPod + } + n.Spec.PodTemplate.Volumes = append(n.Spec.PodTemplate.Volumes, corev1.Volume{ Name: "cache-heater-volume", VolumeSource: corev1.VolumeSource{ @@ -753,9 +770,32 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config }, }) - n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, corev1.VolumeMount{ + cacheHeaterVolume := corev1.VolumeMount{ Name: "cache-heater-volume", MountPath: "/var/cache/cache-heater", + } + + n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, cacheHeaterVolume) + + n.Spec.PodTemplate.InitContainers = append(n.Spec.PodTemplate.InitContainers, corev1.Container{ + Name: "heat-cache", + Image: plan.Spec.Image, + Command: []string{ + initCmd[0], + }, + Args: initCmd[1:], + VolumeMounts: []corev1.VolumeMount{ + cacheHeaterVolume, + { + Name: "cache-vol", + MountPath: plan.Spec.Config.CachePath, + }, + }, + Env: []corev1.EnvVar{ + {Name: "SERVICE_NAME", Value: instance.Namespace}, + {Name: "INSTANCE_NAME", Value: instance.Name}, + {Name: "POD_CMD", Value: rsyncCommandPVCToPod}, + }, }) } @@ -827,7 +867,6 @@ func newHPA(instance v1alpha1.RpaasInstance, nginx nginxv1alpha1.Nginx) autoscal func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *batchv1beta1.CronJob { cronName := instance.Name + cacheHeaterCronJobSuffix - podCommand := "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" schedule := defaultCacheHeaterSchedule if plan.Spec.Config.CacheHeaterSync.Schedule != "" { @@ -839,9 +878,9 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat image = plan.Spec.Config.CacheHeaterSync.Image } - cmds := defaultCacheHeaterCmds - if len(plan.Spec.Config.CacheHeaterSync.Cmds) > 0 { - cmds = plan.Spec.Config.CacheHeaterSync.Cmds + cmds := defaultCacheHeaterCmdPodToPVC + if len(plan.Spec.Config.CacheHeaterSync.CmdPodToPVC) > 0 { + cmds = plan.Spec.Config.CacheHeaterSync.CmdPodToPVC } return &batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ @@ -885,7 +924,7 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat Env: []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: instance.Namespace}, {Name: "INSTANCE_NAME", Value: instance.Name}, - {Name: "POD_CMD", Value: podCommand}, + {Name: "POD_CMD", Value: rsyncCommandPodToPVC}, }, }, }, diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index fafb3df69..6ce066aba 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -1578,6 +1578,7 @@ func TestReconcile(t *testing.T) { Namespace: "default", }, Spec: v1alpha1.RpaasPlanSpec{ + Image: "tsuru:mynginx:test", Config: v1alpha1.NginxConfig{ CacheEnabled: v1alpha1.Bool(true), CacheSize: resourceMustParsePtr("100M"), @@ -1585,14 +1586,20 @@ func TestReconcile(t *testing.T) { CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, + CachePath: "/var/cache/nginx", CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ Schedule: "1 * * * *", Image: "test/test:latest", - Cmds: []string{ + CmdPodToPVC: []string{ "/bin/bash", "-c", "echo 'this is a test'", }, + CmdPVCToPod: []string{ + "/bin/bash", + "-c", + "echo 'this is a the first pod sync'", + }, }, }, }, @@ -1623,6 +1630,23 @@ func TestReconcile(t *testing.T) { assert.Equal(t, resource.MustParse("100M"), *nginx.Spec.Cache.Size) + initContainer := nginx.Spec.PodTemplate.InitContainers[0] + assert.Equal(t, "heat-cache", initContainer.Name) + assert.Equal(t, "tsuru:mynginx:test", initContainer.Image) + assert.Equal(t, "/bin/bash", initContainer.Command[0]) + assert.Equal(t, "-c", initContainer.Args[0]) + assert.Equal(t, "echo 'this is a the first pod sync'", initContainer.Args[1]) + assert.Equal(t, []corev1.EnvVar{ + {Name: "SERVICE_NAME", Value: "default"}, + {Name: "INSTANCE_NAME", Value: "my-instance"}, + {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-heater/nginx /var/cache/nginx/rpaas"}, + }, initContainer.Env) + + assert.Equal(t, []corev1.VolumeMount{ + {Name: "cache-heater-volume", MountPath: "/var/cache/cache-heater"}, + {Name: "cache-vol", MountPath: "/var/cache/nginx"}, + }, initContainer.VolumeMounts) + cronJob := &batchv1beta1.CronJob{} err = client.Get(context.TODO(), types.NamespacedName{Name: "my-instance-heater-cron-job", Namespace: rpaas.Namespace}, cronJob) require.NoError(t, err) From f07b603fb312c567f4cedca5fcb9d3c12eac25d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 24 Apr 2020 17:29:31 -0300 Subject: [PATCH 30/47] Fix command of rsync --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 973ceb7d0..ab1ed76dc 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -75,7 +75,7 @@ done ` mkdir -p /var/cache/cache-heater/temp; mkdir -p /var/cache/nginx/rpaas/nginx_tmp; -bash -c ${POD_CMD} +bash -c "${POD_CMD}" `} ) From 13f2b21be12f63e1a61654ba0953bf96d2c511c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 27 Apr 2020 17:05:10 -0300 Subject: [PATCH 31/47] Avoid hard-coded paths in controller --- .../rpaasinstance/rpaasinstance_controller.go | 117 +++++++++++------- .../rpaasinstance_controller_test.go | 4 + 2 files changed, 73 insertions(+), 48 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index ab1ed76dc..4e20ce566 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -1,4 +1,4 @@ -// Copyright 2020 tsuru authors. All rights reserved. +// Copyright 2019 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -11,6 +11,7 @@ import ( "fmt" "reflect" "sort" + "strings" "text/template" "github.com/imdario/mergo" @@ -46,13 +47,15 @@ const ( defaultCacheHeaterCronImage = "bitnami/kubectl:latest" defaultCacheHeaterSchedule = "* * * * *" defaultPortAllocationResource = "default" - volumeTeamLabel = "tsuru.io/volume-name" + volumeTeamLabel = "tsuru.io/volume-team" cacheHeaterCronJobSuffix = "-heater-cron-job" cacheHeaterVolumeSuffix = "-heater-volume" - rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater" - rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-heater/nginx /var/cache/nginx/rpaas" + cacheHeaterMountPoint = "/var/cache/cache-heater" + + rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=${CACHE_HEATER_MOUNTPOINT}/temp ${CACHE_PATH}/rpaas/nginx ${CACHE_HEATER_MOUNTPOINT}" + rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=${CACHE_PATH}/rpaas/nginx_tmp ${CACHE_HEATER_MOUNTPOINT}/nginx ${CACHE_PATH}/rpaas" ) var ( @@ -73,8 +76,8 @@ done "/bin/bash", "-c", ` -mkdir -p /var/cache/cache-heater/temp; -mkdir -p /var/cache/nginx/rpaas/nginx_tmp; +mkdir -p ${CACHE_HEATER_MOUNTPOINT}/temp; +mkdir -p ${CACHE_PATH}/rpaas/nginx_tmp; bash -c "${POD_CMD}" `} ) @@ -483,6 +486,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1. return r.client.Create(context.TODO(), newestCronJob) } + newestCronJob.ObjectMeta.ResourceVersion = foundCronJob.ObjectMeta.ResourceVersion if !reflect.DeepEqual(foundCronJob.Spec, newestCronJob.Spec) { return r.client.Update(context.TODO(), newestCronJob) } @@ -755,49 +759,50 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config }, } - if plan.Spec.Config.CacheHeaterEnabled { - initCmd := defaultCacheHeaterCmdPVCToPod - if len(plan.Spec.Config.CacheHeaterSync.CmdPVCToPod) > 0 { - initCmd = plan.Spec.Config.CacheHeaterSync.CmdPVCToPod - } + if !plan.Spec.Config.CacheHeaterEnabled { + return n + } - n.Spec.PodTemplate.Volumes = append(n.Spec.PodTemplate.Volumes, corev1.Volume{ - Name: "cache-heater-volume", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Name + cacheHeaterVolumeSuffix, - }, + initCmd := defaultCacheHeaterCmdPVCToPod + if len(plan.Spec.Config.CacheHeaterSync.CmdPVCToPod) > 0 { + initCmd = plan.Spec.Config.CacheHeaterSync.CmdPVCToPod + } + + n.Spec.PodTemplate.Volumes = append(n.Spec.PodTemplate.Volumes, corev1.Volume{ + Name: "cache-heater-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: instance.Name + cacheHeaterVolumeSuffix, }, - }) + }, + }) - cacheHeaterVolume := corev1.VolumeMount{ - Name: "cache-heater-volume", - MountPath: "/var/cache/cache-heater", - } + cacheHeaterVolume := corev1.VolumeMount{ + Name: "cache-heater-volume", + MountPath: cacheHeaterMountPoint, + } - n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, cacheHeaterVolume) + n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, cacheHeaterVolume) - n.Spec.PodTemplate.InitContainers = append(n.Spec.PodTemplate.InitContainers, corev1.Container{ - Name: "heat-cache", - Image: plan.Spec.Image, - Command: []string{ - initCmd[0], - }, - Args: initCmd[1:], - VolumeMounts: []corev1.VolumeMount{ - cacheHeaterVolume, - { - Name: "cache-vol", - MountPath: plan.Spec.Config.CachePath, - }, - }, - Env: []corev1.EnvVar{ - {Name: "SERVICE_NAME", Value: instance.Namespace}, - {Name: "INSTANCE_NAME", Value: instance.Name}, - {Name: "POD_CMD", Value: rsyncCommandPVCToPod}, + n.Spec.PodTemplate.InitContainers = append(n.Spec.PodTemplate.InitContainers, corev1.Container{ + Name: "heat-cache", + Image: plan.Spec.Image, + Command: []string{ + initCmd[0], + }, + Args: initCmd[1:], + VolumeMounts: []corev1.VolumeMount{ + cacheHeaterVolume, + { + Name: "cache-vol", + MountPath: plan.Spec.Config.CachePath, }, - }) - } + }, + Env: append(cacheHeaterEnvVars(instance, plan), corev1.EnvVar{ + Name: "POD_CMD", + Value: interpolateCacheHeaterPodCmdTemplate(rsyncCommandPVCToPod, plan), + }), + }) return n } @@ -921,11 +926,10 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat cmds[0], }, Args: cmds[1:], - Env: []corev1.EnvVar{ - {Name: "SERVICE_NAME", Value: instance.Namespace}, - {Name: "INSTANCE_NAME", Value: instance.Name}, - {Name: "POD_CMD", Value: rsyncCommandPodToPVC}, - }, + Env: append(cacheHeaterEnvVars(instance, plan), corev1.EnvVar{ + Name: "POD_CMD", + Value: interpolateCacheHeaterPodCmdTemplate(rsyncCommandPodToPVC, plan), + }), }, }, RestartPolicy: corev1.RestartPolicyNever, @@ -937,6 +941,23 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat } } +func interpolateCacheHeaterPodCmdTemplate(podCmd string, plan *v1alpha1.RpaasPlan) string { + replacer := strings.NewReplacer( + "${CACHE_HEATER_MOUNTPOINT}", cacheHeaterMountPoint, + "${CACHE_PATH}", plan.Spec.Config.CachePath, + ) + return replacer.Replace(podCmd) +} + +func cacheHeaterEnvVars(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) []corev1.EnvVar { + return []corev1.EnvVar{ + {Name: "SERVICE_NAME", Value: instance.Namespace}, + {Name: "INSTANCE_NAME", Value: instance.Name}, + {Name: "CACHE_HEATER_MOUNTPOINT", Value: cacheHeaterMountPoint}, + {Name: "CACHE_PATH", Value: plan.Spec.Config.CachePath}, + } +} + func shouldDeleteOldConfig(instance *v1alpha1.RpaasInstance, configList *corev1.ConfigMapList) bool { limit := defaultConfigHistoryLimit diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 6ce066aba..2e8c30614 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -1639,6 +1639,8 @@ func TestReconcile(t *testing.T) { assert.Equal(t, []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, + {Name: "CACHE_HEATER_MOUNTPOINT", Value: "/var/cache/cache-heater"}, + {Name: "CACHE_PATH", Value: "/var/cache/nginx"}, {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-heater/nginx /var/cache/nginx/rpaas"}, }, initContainer.Env) @@ -1663,6 +1665,8 @@ func TestReconcile(t *testing.T) { assert.Equal(t, []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, + {Name: "CACHE_HEATER_MOUNTPOINT", Value: "/var/cache/cache-heater"}, + {Name: "CACHE_PATH", Value: "/var/cache/nginx"}, {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater"}, }, podSpec.Containers[0].Env) } From dcb447a8cb938f1fbd4ad72f0885cbc191e9f00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 27 Apr 2020 17:14:22 -0300 Subject: [PATCH 32/47] Avoid to open another bash process --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 4e20ce566..1e0b3b6b4 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -78,7 +78,7 @@ done ` mkdir -p ${CACHE_HEATER_MOUNTPOINT}/temp; mkdir -p ${CACHE_PATH}/rpaas/nginx_tmp; -bash -c "${POD_CMD}" +${POD_CMD} `} ) From f92d0348f7735188e41b0752d5f51e52c428f34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 27 Apr 2020 17:21:10 -0300 Subject: [PATCH 33/47] Convert to nginx quantity always using bytes --- internal/pkg/rpaas/nginx/configuration_render.go | 10 +--------- internal/pkg/rpaas/nginx/configuration_render_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/internal/pkg/rpaas/nginx/configuration_render.go b/internal/pkg/rpaas/nginx/configuration_render.go index 9113232f2..ee07a704a 100644 --- a/internal/pkg/rpaas/nginx/configuration_render.go +++ b/internal/pkg/rpaas/nginx/configuration_render.go @@ -16,8 +16,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) -var terabyte = resource.NewQuantity(1*1024*1024*1024*1024, resource.BinarySI) - type ConfigurationRenderer interface { Render(ConfigurationData) (string, error) } @@ -155,13 +153,7 @@ func k8sQuantityToNginx(quantity *resource.Quantity) string { } bytesN, _ := quantity.AsInt64() - // nginx is not support terabyte yet - // https://github.com/nginx/nginx/blob/3ba88365b5acef17f01671cd969c909dee5e2cde/src/core/ngx_parse.c#L81-L87 - if quantity.Cmp(*terabyte) >= 0 { - return strconv.Itoa(int(bytesN)) - } - - return strings.TrimSuffix(resource.NewQuantity(bytesN, resource.BinarySI).String(), "i") + return strconv.Itoa(int(bytesN)) } var templateFuncs = template.FuncMap(map[string]interface{}{ diff --git a/internal/pkg/rpaas/nginx/configuration_render_test.go b/internal/pkg/rpaas/nginx/configuration_render_test.go index 95b8339fa..cc4bae3d3 100644 --- a/internal/pkg/rpaas/nginx/configuration_render_test.go +++ b/internal/pkg/rpaas/nginx/configuration_render_test.go @@ -79,7 +79,7 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { Instance: &v1alpha1.RpaasInstance{}, }, assertion: func(t *testing.T, result string) { - assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100M;`, result) + assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:104857600;`, result) assert.Regexp(t, `proxy_temp_path /path/to/cache/dir/nginx_tmp 1 2;`, result) assert.Regexp(t, `server { \s+listen 8800; @@ -105,7 +105,7 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { Instance: &v1alpha1.RpaasInstance{}, }, assertion: func(t *testing.T, result string) { - assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100M inactive=12h max_size=300M loader_files=1000;`, result) + assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:104857600 inactive=12h max_size=314572800 loader_files=1000;`, result) assert.Regexp(t, `proxy_temp_path /path/to/cache/dir/nginx_tmp 1 2;`, result) assert.Regexp(t, `server { \s+listen 8800; @@ -570,10 +570,10 @@ func TestK8sQuantityToNginx(t *testing.T) { } expectations := []expectation{ - {"100Ki", "100K"}, - {"100Mi", "100M"}, + {"100Ki", "102400"}, + {"100Mi", "104857600"}, {"100M", "100000000"}, - {"1Gi", "1G"}, + {"1Gi", "1073741824"}, {"1G", "1000000000"}, {"1024Gi", "1099511627776"}, {"2Ti", "2199023255552"}, From bffe09bac34472dd4448a7594a2de39f247e72a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 27 Apr 2020 18:25:53 -0300 Subject: [PATCH 34/47] Review RBAC, thanks @bernardolins --- deploy/role.yaml | 24 ++++++++++++++++++++++++ deploy/role_binding.yaml | 13 +++++++++++++ deploy/service_account.yaml | 5 +++++ 3 files changed, 42 insertions(+) diff --git a/deploy/role.yaml b/deploy/role.yaml index b98b5b368..7a4b184df 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -60,3 +60,27 @@ rules: - horizontalpodautoscalers verbs: - '*' +- apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rpaas-cache-syncer +rules: +- apiGroups: [""] + resources: + - pods + verbs: + - get + - list +- apiGroups: [""] + resources: + - pods/exec + verbs: + - create diff --git a/deploy/role_binding.yaml b/deploy/role_binding.yaml index aa2c58010..b1f522a57 100644 --- a/deploy/role_binding.yaml +++ b/deploy/role_binding.yaml @@ -10,3 +10,16 @@ roleRef: kind: ClusterRole name: rpaas-operator apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rpaas-cache-heater-cronjob +subjects: +- kind: ServiceAccount + name: rpaas-cache-heater + namespace: rpaas-operator-integration +roleRef: + kind: ClusterRole + name: rpaas-cache-syncer + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/service_account.yaml b/deploy/service_account.yaml index 9b0b0dde3..a869a8b01 100644 --- a/deploy/service_account.yaml +++ b/deploy/service_account.yaml @@ -2,3 +2,8 @@ apiVersion: v1 kind: ServiceAccount metadata: name: rpaas-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rpaas-cache-heater From 21227c5112f5aedea3e5d2b03862d235bb04930d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 28 Apr 2020 10:38:59 -0300 Subject: [PATCH 35/47] Fix notation of testdata --- test/testdata/rpaas-full.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testdata/rpaas-full.yaml b/test/testdata/rpaas-full.yaml index 91d9558f7..2693cba40 100644 --- a/test/testdata/rpaas-full.yaml +++ b/test/testdata/rpaas-full.yaml @@ -7,8 +7,8 @@ spec: config: cacheEnabled: true cachePath: /var/cache/nginx - cacheZoneSize: 100m - cacheSize: 300m + cacheZoneSize: 100Mi + cacheSize: 300Mi cacheHeaterEnabled: false resources: limits: From efa3c592b74fd71253c132fa371aeab5345593c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 28 Apr 2020 11:36:29 -0300 Subject: [PATCH 36/47] Disable cache on integration tests --- test/testdata/rpaas-full.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/testdata/rpaas-full.yaml b/test/testdata/rpaas-full.yaml index 2693cba40..942f46dbe 100644 --- a/test/testdata/rpaas-full.yaml +++ b/test/testdata/rpaas-full.yaml @@ -4,12 +4,7 @@ metadata: name: basic spec: image: tsuru/nginx-tsuru:1.16.1 - config: - cacheEnabled: true - cachePath: /var/cache/nginx - cacheZoneSize: 100Mi - cacheSize: 300Mi - cacheHeaterEnabled: false + config: {} resources: limits: memory: "128Mi" From e37279688cdeb49951fbd7d57c039ad2af5de5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 29 Apr 2020 10:58:58 -0300 Subject: [PATCH 37/47] Rename deprecated image name: `gcr.io/hello-minikube-zero-install/hello-node` --- test/integration_test.go | 48 +++++++++++++----------------------- test/testdata/hello-app.yaml | 2 +- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/test/integration_test.go b/test/integration_test.go index b590bcada..c89779caa 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -36,6 +36,16 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +func assertInstanceContains(t *testing.T, localPort int, expectedStatus int, bodyPart string) { + rsp, iErr := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", localPort)) + require.NoError(t, iErr) + assert.Equal(t, expectedStatus, rsp.StatusCode) + defer rsp.Body.Close() + rawBody, iErr := ioutil.ReadAll(rsp.Body) + require.NoError(t, iErr) + assert.Contains(t, string(rawBody), bodyPart) +} + func Test_RpaasOperator(t *testing.T) { t.Run("apply manifests at rpaas-full.yaml", func(t *testing.T) { namespaceName := "rpaasoperator-full" + strconv.Itoa(rand.Int()) @@ -213,22 +223,13 @@ func Test_RpaasApi(t *testing.T) { _, err = kubectl("wait", "--for=condition=Ready", "-l", "app=hello", "pod", "--timeout", "2m", "-n", namespaceName) require.NoError(t, err) - assertInstanceReturns := func(localPort int, expectedBody string) { - rsp, iErr := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", localPort)) - require.NoError(t, iErr) - defer rsp.Body.Close() - rawBody, iErr := ioutil.ReadAll(rsp.Body) - require.NoError(t, iErr) - assert.Equal(t, expectedBody, string(rawBody)) - } - serviceName := fmt.Sprintf("svc/%s-service", instanceName) servicePort := "80" ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturns(localPort, "instance not bound\n") + assertInstanceContains(t, localPort, http.StatusNotFound, "instance not bound") }) require.NoError(t, err) @@ -244,7 +245,7 @@ func Test_RpaasApi(t *testing.T) { ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturns(localPort, "Hello World!") + assertInstanceContains(t, localPort, http.StatusOK, "CLIENT VALUES") }) require.NoError(t, err) @@ -259,7 +260,7 @@ func Test_RpaasApi(t *testing.T) { ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturns(localPort, "instance not bound\n") + assertInstanceContains(t, localPort, http.StatusNotFound, "instance not bound") }) require.NoError(t, err) }) @@ -294,22 +295,13 @@ func Test_RpaasApi(t *testing.T) { _, err = kubectl("wait", "--for=condition=Ready", "-l", "app=echo-server", "pod", "--timeout", "2m", "-n", namespaceName) require.NoError(t, err) - assertInstanceReturns := func(localPort int, expectedBody string) { - rsp, iErr := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", localPort)) - require.NoError(t, iErr) - defer rsp.Body.Close() - rawBody, iErr := ioutil.ReadAll(rsp.Body) - require.NoError(t, iErr) - assert.Equal(t, expectedBody, string(rawBody)) - } - serviceName := fmt.Sprintf("svc/%s-service", instanceName) servicePort := "80" ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturns(localPort, "instance not bound\n") + assertInstanceContains(t, localPort, http.StatusNotFound, "instance not bound") }) require.NoError(t, err) @@ -328,7 +320,7 @@ func Test_RpaasApi(t *testing.T) { ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturns(localPort, "Hello World!") + assertInstanceContains(t, localPort, http.StatusOK, "CLIENT VALUES") }) require.NoError(t, err) @@ -340,16 +332,10 @@ func Test_RpaasApi(t *testing.T) { _, err = getReadyNginx(instanceName, namespaceName, 1, 1) require.NoError(t, err) - assertInstanceReturnsStatusCode := func(localPort int, expectedCode int) { - rsp, iErr := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", localPort)) - require.NoError(t, iErr) - assert.Equal(t, expectedCode, rsp.StatusCode) - } - ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturnsStatusCode(localPort, http.StatusOK) + assertInstanceContains(t, localPort, http.StatusOK, "") }) require.NoError(t, err) @@ -364,7 +350,7 @@ func Test_RpaasApi(t *testing.T) { ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() err = portForward(ctx, namespaceName, serviceName, servicePort, func(localPort int) { - assertInstanceReturns(localPort, "instance not bound\n") + assertInstanceContains(t, localPort, http.StatusNotFound, "instance not bound") }) require.NoError(t, err) diff --git a/test/testdata/hello-app.yaml b/test/testdata/hello-app.yaml index 4dd44636c..13e27bcaf 100644 --- a/test/testdata/hello-app.yaml +++ b/test/testdata/hello-app.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: hello - image: gcr.io/hello-minikube-zero-install/hello-node + image: k8s.gcr.io/echoserver:1.4 --- apiVersion: v1 kind: Service From 712afa223abda1c0bb92ebd144cfbee82da04321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 7 May 2020 09:46:57 -0300 Subject: [PATCH 38/47] Update pkg/apis/extensions/v1alpha1/rpaasplan_types.go Co-authored-by: Claudio Netto --- pkg/apis/extensions/v1alpha1/rpaasplan_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index eefc82f95..a590b71ea 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -87,7 +87,7 @@ type NginxConfig struct { } type CacheHeaterSyncSpec struct { - // The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. + // Schedule is the the cron time string format, see https://en.wikipedia.org/wiki/Cron. Schedule string `json:"schedule,omitempty"` // Container image used to sync the containers From 6bafebee03ad9e355f7bee3aa290262963081f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 7 May 2020 10:09:30 -0300 Subject: [PATCH 39/47] Advocate use of context.Context --- .../rpaasinstance/rpaasinstance_controller.go | 124 +++++++++--------- .../rpaasinstance_controller_test.go | 30 +++-- 2 files changed, 82 insertions(+), 72 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 1e0b3b6b4..f402a0344 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -208,49 +208,33 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile } } - rendered, err := r.renderTemplate(instance, plan) + rendered, err := r.renderTemplate(ctx, instance, plan) if err != nil { return reconcile.Result{}, err } configMap := newConfigMap(instance, rendered) - err = r.reconcileConfigMap(configMap) + err = r.reconcileConfigMap(ctx, configMap) if err != nil { return reconcile.Result{}, err } - configList, err := r.listConfigs(instance) + configList, err := r.listConfigs(ctx, instance) if err != nil { return reconcile.Result{}, err } if shouldDeleteOldConfig(instance, configList) { - if err = r.deleteOldConfig(instance, configList); err != nil { + if err = r.deleteOldConfig(ctx, instance, configList); err != nil { return reconcile.Result{}, err } } nginx := newNginx(instance, plan, configMap) - if err = r.reconcileNginx(nginx); err != nil { + if err = r.reconcileNginx(ctx, nginx); err != nil { return reconcile.Result{}, err } - if plan.Spec.Config.CacheHeaterEnabled { - err = r.reconcileCacheHeaterCronJob(instance, plan) - if err != nil { - return reconcile.Result{}, err - } - err = r.reconcileCacheHeaterVolume(instance, plan) - if err != nil { - return reconcile.Result{}, err - } - } else { - err = r.destroyCacheHeaterCronJob(instance) - if err != nil { - return reconcile.Result{}, err - } - err = r.destroyCacheHeaterVolume(instance) - if err != nil { - return reconcile.Result{}, err - } + if err = r.reconcileCacheHeater(ctx, instance, plan); err != nil { + return reconcile.Result{}, err } if err = r.reconcileHPA(ctx, *instance, *nginx); err != nil { @@ -285,7 +269,7 @@ func (r *ReconcileRpaasInstance) mergeInstanceWithFlavors(ctx context.Context, i logger := log.WithName("mergeInstanceWithFlavors"). WithValues("RpaasInstance", types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}) - defaultFlavors, err := r.listDefaultFlavors(instance) + defaultFlavors, err := r.listDefaultFlavors(ctx, instance) if err != nil { return nil, err } @@ -350,9 +334,9 @@ func mergeInstanceWithFlavor(instance *v1alpha1.RpaasInstance, flavor v1alpha1.R return nil } -func (r *ReconcileRpaasInstance) listDefaultFlavors(instance *v1alpha1.RpaasInstance) ([]v1alpha1.RpaasFlavor, error) { +func (r *ReconcileRpaasInstance) listDefaultFlavors(ctx context.Context, instance *v1alpha1.RpaasInstance) ([]v1alpha1.RpaasFlavor, error) { flavorList := &v1alpha1.RpaasFlavorList{} - if err := r.client.List(context.TODO(), flavorList, client.InNamespace(instance.Namespace)); err != nil { + if err := r.client.List(ctx, flavorList, client.InNamespace(instance.Namespace)); err != nil { return nil, err } var result []v1alpha1.RpaasFlavor @@ -425,15 +409,15 @@ func (r *ReconcileRpaasInstance) reconcileHPA(ctx context.Context, instance v1al return nil } -func (r *ReconcileRpaasInstance) reconcileConfigMap(configMap *corev1.ConfigMap) error { +func (r *ReconcileRpaasInstance) reconcileConfigMap(ctx context.Context, configMap *corev1.ConfigMap) error { found := &corev1.ConfigMap{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: configMap.ObjectMeta.Name, Namespace: configMap.ObjectMeta.Namespace}, found) + err := r.client.Get(ctx, types.NamespacedName{Name: configMap.ObjectMeta.Name, Namespace: configMap.ObjectMeta.Namespace}, found) if err != nil { if !k8sErrors.IsNotFound(err) { logrus.Errorf("Failed to get configMap: %v", err) return err } - err = r.client.Create(context.TODO(), configMap) + err = r.client.Create(ctx, configMap) if err != nil { logrus.Errorf("Failed to create configMap: %v", err) return err @@ -442,22 +426,22 @@ func (r *ReconcileRpaasInstance) reconcileConfigMap(configMap *corev1.ConfigMap) } configMap.ObjectMeta.ResourceVersion = found.ObjectMeta.ResourceVersion - err = r.client.Update(context.TODO(), configMap) + err = r.client.Update(ctx, configMap) if err != nil { logrus.Errorf("Failed to update configMap: %v", err) } return err } -func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) error { +func (r *ReconcileRpaasInstance) reconcileNginx(ctx context.Context, nginx *nginxv1alpha1.Nginx) error { found := &nginxv1alpha1.Nginx{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: nginx.ObjectMeta.Name, Namespace: nginx.ObjectMeta.Namespace}, found) + err := r.client.Get(ctx, types.NamespacedName{Name: nginx.ObjectMeta.Name, Namespace: nginx.ObjectMeta.Namespace}, found) if err != nil { if !k8sErrors.IsNotFound(err) { logrus.Errorf("Failed to get nginx CR: %v", err) return err } - err = r.client.Create(context.TODO(), nginx) + err = r.client.Create(ctx, nginx) if err != nil { logrus.Errorf("Failed to create nginx CR: %v", err) return err @@ -466,59 +450,74 @@ func (r *ReconcileRpaasInstance) reconcileNginx(nginx *nginxv1alpha1.Nginx) erro } nginx.ObjectMeta.ResourceVersion = found.ObjectMeta.ResourceVersion - err = r.client.Update(context.TODO(), nginx) + err = r.client.Update(ctx, nginx) if err != nil { logrus.Errorf("Failed to update nginx CR: %v", err) } return err } -func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { +func (r *ReconcileRpaasInstance) reconcileCacheHeater(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { + if plan.Spec.Config.CacheHeaterEnabled { + err := r.reconcileCacheHeaterCronJob(ctx, instance, plan) + if err != nil { + return err + } + return r.reconcileCacheHeaterVolume(ctx, instance, plan) + } + + err := r.destroyCacheHeaterCronJob(ctx, instance) + if err != nil { + return err + } + return r.destroyCacheHeaterVolume(ctx, instance) +} + +func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { foundCronJob := &batchv1beta1.CronJob{} cronName := instance.Name + cacheHeaterCronJobSuffix - err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, foundCronJob) + err := r.client.Get(ctx, types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, foundCronJob) if err != nil && !k8sErrors.IsNotFound(err) { return err } newestCronJob := newCronJob(instance, plan) if k8sErrors.IsNotFound(err) { - return r.client.Create(context.TODO(), newestCronJob) + return r.client.Create(ctx, newestCronJob) } newestCronJob.ObjectMeta.ResourceVersion = foundCronJob.ObjectMeta.ResourceVersion if !reflect.DeepEqual(foundCronJob.Spec, newestCronJob.Spec) { - return r.client.Update(context.TODO(), newestCronJob) + return r.client.Update(ctx, newestCronJob) } return nil } -func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(instance *v1alpha1.RpaasInstance) error { +func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(ctx context.Context, instance *v1alpha1.RpaasInstance) error { cronName := instance.Name + cacheHeaterCronJobSuffix cronJob := &batchv1beta1.CronJob{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) + err := r.client.Get(ctx, types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) isNotFound := k8sErrors.IsNotFound(err) if err != nil && !isNotFound { return err } else if isNotFound { - logrus.Infof("CronJob %s is not found, skipping destruction", cronName) return nil } - return r.client.Delete(context.TODO(), cronJob) + logrus.Infof("deleting cronjob %s", cronName) + return r.client.Delete(ctx, cronJob) } -func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { +func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { pvcName := instance.Name + cacheHeaterVolumeSuffix pvc := &corev1.PersistentVolumeClaim{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) + err := r.client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) isNotFound := k8sErrors.IsNotFound(err) if err != nil && !isNotFound { return err } else if !isNotFound { - logrus.Infof("PersistentVolumeClaim %s is found, skipping creation", pvcName) return nil } @@ -571,35 +570,36 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(instance *v1alpha1.R } } - if err := r.client.Create(context.TODO(), pvc); err != nil { + logrus.Infof("creating PersistentVolumeClaim %s", pvcName) + if err := r.client.Create(ctx, pvc); err != nil { return err } return nil } -func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(instance *v1alpha1.RpaasInstance) error { +func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(ctx context.Context, instance *v1alpha1.RpaasInstance) error { pvcName := instance.Name + cacheHeaterVolumeSuffix pvc := &corev1.PersistentVolumeClaim{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) + err := r.client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) isNotFound := k8sErrors.IsNotFound(err) if err != nil && !isNotFound { return err } else if isNotFound { - logrus.Infof("PersistentVolumeClaim %s is not found, skipping destruction", pvcName) return nil } - return r.client.Delete(context.TODO(), pvc) + logrus.Infof("deleting PersistentVolumeClaim %s", pvcName) + return r.client.Delete(ctx, pvc) } -func (r *ReconcileRpaasInstance) renderTemplate(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) (string, error) { - blocks, err := r.getConfigurationBlocks(instance, plan) +func (r *ReconcileRpaasInstance) renderTemplate(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) (string, error) { + blocks, err := r.getConfigurationBlocks(ctx, instance, plan) if err != nil { return "", err } - if err = r.updateLocationValues(instance); err != nil { + if err = r.updateLocationValues(ctx, instance); err != nil { return "", err } @@ -614,11 +614,11 @@ func (r *ReconcileRpaasInstance) renderTemplate(instance *v1alpha1.RpaasInstance }) } -func (r *ReconcileRpaasInstance) getConfigurationBlocks(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) (nginx.ConfigurationBlocks, error) { +func (r *ReconcileRpaasInstance) getConfigurationBlocks(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) (nginx.ConfigurationBlocks, error) { var blocks nginx.ConfigurationBlocks if plan.Spec.Template != nil { - mainBlock, err := util.GetValue(context.TODO(), r.client, "", plan.Spec.Template) + mainBlock, err := util.GetValue(ctx, r.client, "", plan.Spec.Template) if err != nil { return blocks, err } @@ -627,7 +627,7 @@ func (r *ReconcileRpaasInstance) getConfigurationBlocks(instance *v1alpha1.Rpaas } for blockType, blockValue := range instance.Spec.Blocks { - content, err := util.GetValue(context.TODO(), r.client, instance.Namespace, &blockValue) + content, err := util.GetValue(ctx, r.client, instance.Namespace, &blockValue) if err != nil { return blocks, err } @@ -649,13 +649,13 @@ func (r *ReconcileRpaasInstance) getConfigurationBlocks(instance *v1alpha1.Rpaas return blocks, nil } -func (r *ReconcileRpaasInstance) updateLocationValues(instance *v1alpha1.RpaasInstance) error { +func (r *ReconcileRpaasInstance) updateLocationValues(ctx context.Context, instance *v1alpha1.RpaasInstance) error { for _, location := range instance.Spec.Locations { if location.Content == nil { continue } - content, err := util.GetValue(context.TODO(), r.client, instance.Namespace, location.Content) + content, err := util.GetValue(ctx, r.client, instance.Namespace, location.Content) if err != nil { return err } @@ -665,7 +665,7 @@ func (r *ReconcileRpaasInstance) updateLocationValues(instance *v1alpha1.RpaasIn return nil } -func (r *ReconcileRpaasInstance) listConfigs(instance *v1alpha1.RpaasInstance) (*corev1.ConfigMapList, error) { +func (r *ReconcileRpaasInstance) listConfigs(ctx context.Context, instance *v1alpha1.RpaasInstance) (*corev1.ConfigMapList, error) { configList := &corev1.ConfigMapList{} listOptions := &client.ListOptions{Namespace: instance.ObjectMeta.Namespace} client.MatchingLabels(map[string]string{ @@ -673,16 +673,16 @@ func (r *ReconcileRpaasInstance) listConfigs(instance *v1alpha1.RpaasInstance) ( "type": "config", }).ApplyToList(listOptions) - err := r.client.List(context.TODO(), configList, listOptions) + err := r.client.List(ctx, configList, listOptions) return configList, err } -func (r *ReconcileRpaasInstance) deleteOldConfig(instance *v1alpha1.RpaasInstance, configList *corev1.ConfigMapList) error { +func (r *ReconcileRpaasInstance) deleteOldConfig(ctx context.Context, instance *v1alpha1.RpaasInstance, configList *corev1.ConfigMapList) error { list := configList.Items sort.Slice(list, func(i, j int) bool { return list[i].ObjectMeta.CreationTimestamp.String() < list[j].ObjectMeta.CreationTimestamp.String() }) - if err := r.client.Delete(context.TODO(), &list[0]); err != nil { + if err := r.client.Delete(ctx, &list[0]); err != nil { return err } return nil diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 2e8c30614..d3bcd99e1 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -659,6 +659,7 @@ func Test_reconcileHPA(t *testing.T) { } func Test_reconcileHeaterVolume(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -682,7 +683,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -696,6 +697,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { } func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -723,7 +725,7 @@ func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -736,6 +738,7 @@ func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { } func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" instance1.SetTeamOwner("team-one") @@ -764,7 +767,7 @@ func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -778,6 +781,7 @@ func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { } func Test_reconcileHeaterVolumeLabels(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" instance1.SetTeamOwner("team-one") @@ -807,7 +811,7 @@ func Test_reconcileHeaterVolumeLabels(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -821,6 +825,7 @@ func Test_reconcileHeaterVolumeLabels(t *testing.T) { } func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -845,7 +850,7 @@ func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -858,6 +863,7 @@ func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { } func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -883,7 +889,7 @@ func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterVolume(instance1, plan) + err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} @@ -896,6 +902,7 @@ func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { } func Test_destroyHeaterVolume(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -915,7 +922,7 @@ func Test_destroyHeaterVolume(t *testing.T) { scheme: newScheme(), } - err := reconciler.destroyCacheHeaterVolume(instance1) + err := reconciler.destroyCacheHeaterVolume(ctx, instance1) require.NoError(t, err) pvc = &corev1.PersistentVolumeClaim{} @@ -924,6 +931,7 @@ func Test_destroyHeaterVolume(t *testing.T) { } func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -942,7 +950,7 @@ func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{}, } - err := reconciler.reconcileCacheHeaterCronJob(instance1, plan) + err := reconciler.reconcileCacheHeaterCronJob(ctx, instance1, plan) require.NoError(t, err) cronJob := &batchv1beta1.CronJob{} @@ -954,6 +962,7 @@ func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { } func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -987,7 +996,7 @@ func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { }, } - err := reconciler.reconcileCacheHeaterCronJob(instance1, plan) + err := reconciler.reconcileCacheHeaterCronJob(ctx, instance1, plan) require.NoError(t, err) cronJob := &batchv1beta1.CronJob{} @@ -1000,6 +1009,7 @@ func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { } func Test_destroyHeaterCronJob(t *testing.T) { + ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -1021,7 +1031,7 @@ func Test_destroyHeaterCronJob(t *testing.T) { scheme: newScheme(), } - err := reconciler.destroyCacheHeaterCronJob(instance1) + err := reconciler.destroyCacheHeaterCronJob(ctx, instance1) require.NoError(t, err) cronJob = &batchv1beta1.CronJob{} From 0fa1c1587ec53d293bf91d51a4458b955249df91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 7 May 2020 10:12:06 -0300 Subject: [PATCH 40/47] Add header license --- .../extensions/v1alpha1/rpaasinstance_test.go | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go b/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go index 8261151e5..cb1bc1bda 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go +++ b/pkg/apis/extensions/v1alpha1/rpaasinstance_test.go @@ -1,31 +1,35 @@ +// Copyright 2020 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package v1alpha1 -import( +import ( "testing" "github.com/stretchr/testify/assert" ) func Test_SetTeamOwner(t *testing.T) { - instance := &RpaasInstance{} - instance.SetTeamOwner("team-one") - expected := map[string]string{teamOwnerLabel: "team-one"} - assert.Equal(t, expected, instance.Labels) - assert.Equal(t, expected, instance.Annotations) - assert.Equal(t, expected, instance.Spec.PodTemplate.Labels) + instance := &RpaasInstance{} + instance.SetTeamOwner("team-one") + expected := map[string]string{teamOwnerLabel: "team-one"} + assert.Equal(t, expected, instance.Labels) + assert.Equal(t, expected, instance.Annotations) + assert.Equal(t, expected, instance.Spec.PodTemplate.Labels) - instance.SetTeamOwner("team-two") - expected = map[string]string{teamOwnerLabel: "team-two"} - assert.Equal(t, expected, instance.Labels) - assert.Equal(t, expected, instance.Annotations) - assert.Equal(t, expected, instance.Spec.PodTemplate.Labels) + instance.SetTeamOwner("team-two") + expected = map[string]string{teamOwnerLabel: "team-two"} + assert.Equal(t, expected, instance.Labels) + assert.Equal(t, expected, instance.Annotations) + assert.Equal(t, expected, instance.Spec.PodTemplate.Labels) } func Test_GetTeamOwner(t *testing.T) { - instance := &RpaasInstance{} - owner := instance.TeamOwner() - assert.Equal(t, "", owner) - instance.SetTeamOwner("team-one") - owner = instance.TeamOwner() - assert.Equal(t, "team-one", owner) + instance := &RpaasInstance{} + owner := instance.TeamOwner() + assert.Equal(t, "", owner) + instance.SetTeamOwner("team-one") + owner = instance.TeamOwner() + assert.Equal(t, "team-one", owner) } From 5ccb0a22c0daed18fcbc2f4aabcb380fcfd11a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 7 May 2020 11:06:59 -0300 Subject: [PATCH 41/47] Use table driven tests to reconsileCacheVolume --- .../extensions/v1alpha1/rpaasplan_types.go | 6 +- .../rpaasinstance_controller_test.go | 298 +++++------------- 2 files changed, 88 insertions(+), 216 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index a590b71ea..b716f615e 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -90,12 +90,14 @@ type CacheHeaterSyncSpec struct { // Schedule is the the cron time string format, see https://en.wikipedia.org/wiki/Cron. Schedule string `json:"schedule,omitempty"` - // Container image used to sync the containers + // Container is the image used to sync the containers // default is bitnami/kubectl:latest Image string `json:"image,omitempty"` - // Cmds that are used to customize command used to sync memory cache to persistent storage + // CmdPodToPVC is used to customize command used to sync memory cache (POD) to persistent storage (PVC) CmdPodToPVC []string `json:"cmdPodToPVC,omitempty"` + + // CmdPVCToPod is used to customize command used to sync persistent storage (PVC) to memory cache (POD) CmdPVCToPod []string `json:"cmdPVCToPod,omitempty"` } diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index d3bcd99e1..def231163 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -660,245 +660,115 @@ func Test_reconcileHPA(t *testing.T) { func Test_reconcileHeaterVolume(t *testing.T) { ctx := context.TODO() - instance1 := newEmptyRpaasInstance() - instance1.Name = "instance-1" - - resources := []runtime.Object{} - scheme := newScheme() - corev1.AddToScheme(scheme) - - k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) - reconciler := &ReconcileRpaasInstance{ - client: k8sClient, - scheme: newScheme(), - } + rpaasInstance := newEmptyRpaasInstance() + rpaasInstance.Name = "my-instance" + rpaasInstance.SetTeamOwner("team-one") - plan := &v1alpha1.RpaasPlan{ - Spec: v1alpha1.RpaasPlanSpec{ - Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), + tests := []struct { + name string + planSpec v1alpha1.RpaasPlanSpec + assert func(*testing.T, *corev1.PersistentVolumeClaim) + }{ + { + name: "Should repass attributes to PVC", + planSpec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheSize: resourceMustParsePtr("10Gi"), + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + }, }, }, - }, - } + assert: func(t *testing.T, pvc *corev1.PersistentVolumeClaim) { + assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Kind, "RpaasInstance") + assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Name, rpaasInstance.Name) + assert.Equal(t, pvc.Spec.StorageClassName, strPtr("my-storage-class")) + assert.Equal(t, pvc.Spec.AccessModes, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}) - err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) - require.NoError(t, err) - - pvc := &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) - require.NoError(t, err) - - assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Kind, "RpaasInstance") - assert.Equal(t, pvc.ObjectMeta.OwnerReferences[0].Name, instance1.Name) - assert.Equal(t, pvc.Spec.StorageClassName, strPtr("my-storage-class")) - assert.Equal(t, pvc.Spec.AccessModes, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}) -} - -func Test_reconcileHeaterVolumeWithLabels(t *testing.T) { - ctx := context.TODO() - instance1 := newEmptyRpaasInstance() - instance1.Name = "instance-1" - - resources := []runtime.Object{} - scheme := newScheme() - corev1.AddToScheme(scheme) - - k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) - reconciler := &ReconcileRpaasInstance{ - client: k8sClient, - scheme: newScheme(), - } - - plan := &v1alpha1.RpaasPlan{ - Spec: v1alpha1.RpaasPlanSpec{ - Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), - VolumeLabels: map[string]string{ - "some-label": "foo", - "other-label": "bar", - }, - }, + parsedSize, _ := resource.ParseQuantity("10Gi") + assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) }, }, - } - - err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) - require.NoError(t, err) - - pvc := &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) - require.NoError(t, err) - - assert.Equal(t, 2, len(pvc.ObjectMeta.Labels)) - assert.Equal(t, "foo", pvc.ObjectMeta.Labels["some-label"]) - assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) -} - -func Test_reconcileHeaterVolumeWithInstanceTeamOwner(t *testing.T) { - ctx := context.TODO() - instance1 := newEmptyRpaasInstance() - instance1.Name = "instance-1" - instance1.SetTeamOwner("team-one") - - resources := []runtime.Object{} - scheme := newScheme() - corev1.AddToScheme(scheme) - - k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) - reconciler := &ReconcileRpaasInstance{ - client: k8sClient, - scheme: newScheme(), - } - - plan := &v1alpha1.RpaasPlan{ - Spec: v1alpha1.RpaasPlanSpec{ - Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), - VolumeLabels: map[string]string{ - "some-label": "foo", - "other-label": "bar", + { + name: "Should repass volume labels to PVC", + planSpec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + StorageClassName: strPtr("my-storage-class"), + VolumeLabels: map[string]string{ + "some-label": "foo", + "other-label": "bar", + }, }, }, }, + assert: func(t *testing.T, pvc *corev1.PersistentVolumeClaim) { + assert.Equal(t, 3, len(pvc.ObjectMeta.Labels)) + assert.Equal(t, map[string]string{ + "some-label": "foo", + "other-label": "bar", + "tsuru.io/volume-team": "team-one", + }, pvc.ObjectMeta.Labels) + }, }, - } - - err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) - require.NoError(t, err) - - pvc := &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) - require.NoError(t, err) - assert.Equal(t, 3, len(pvc.ObjectMeta.Labels)) - assert.Equal(t, "foo", pvc.ObjectMeta.Labels["some-label"]) - assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) - assert.Equal(t, "team-one", pvc.ObjectMeta.Labels[volumeTeamLabel]) -} - -func Test_reconcileHeaterVolumeLabels(t *testing.T) { - ctx := context.TODO() - instance1 := newEmptyRpaasInstance() - instance1.Name = "instance-1" - instance1.SetTeamOwner("team-one") - - resources := []runtime.Object{} - scheme := newScheme() - corev1.AddToScheme(scheme) - - k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) - reconciler := &ReconcileRpaasInstance{ - client: k8sClient, - scheme: newScheme(), - } - - plan := &v1alpha1.RpaasPlan{ - Spec: v1alpha1.RpaasPlanSpec{ - Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), - VolumeLabels: map[string]string{ - "some-label": "foo", - "other-label": "bar", - volumeTeamLabel: "another-team", + { + name: "Should priorize the team inside plan", + planSpec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + VolumeLabels: map[string]string{ + "tsuru.io/volume-team": "another-team", + }, }, }, }, + assert: func(t *testing.T, pvc *corev1.PersistentVolumeClaim) { + assert.Equal(t, "another-team", pvc.ObjectMeta.Labels["tsuru.io/volume-team"]) + }, }, - } - - err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) - require.NoError(t, err) - - pvc := &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) - require.NoError(t, err) - - assert.Equal(t, 3, len(pvc.ObjectMeta.Labels)) - assert.Equal(t, "foo", pvc.ObjectMeta.Labels["some-label"]) - assert.Equal(t, "bar", pvc.ObjectMeta.Labels["other-label"]) - assert.Equal(t, "another-team", pvc.ObjectMeta.Labels[volumeTeamLabel]) -} - -func Test_reconcileHeaterVolumeUsingCacheSize(t *testing.T) { - ctx := context.TODO() - instance1 := newEmptyRpaasInstance() - instance1.Name = "instance-1" - - resources := []runtime.Object{} - scheme := newScheme() - corev1.AddToScheme(scheme) - - k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) - reconciler := &ReconcileRpaasInstance{ - client: k8sClient, - scheme: newScheme(), - } - - plan := &v1alpha1.RpaasPlan{ - Spec: v1alpha1.RpaasPlanSpec{ - Config: v1alpha1.NginxConfig{ - CacheSize: resourceMustParsePtr("10Gi"), - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), + { + name: "Should allow to customize size of PVC separately of cache settings", + planSpec: v1alpha1.RpaasPlanSpec{ + Config: v1alpha1.NginxConfig{ + CacheSize: resourceMustParsePtr("10Gi"), + CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + StorageSize: resourceMustParsePtr("100Gi"), + }, }, }, + assert: func(t *testing.T, pvc *corev1.PersistentVolumeClaim) { + parsedSize, _ := resource.ParseQuantity("100Gi") + assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) + }, }, } - err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) - require.NoError(t, err) - - pvc := &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) - require.NoError(t, err) - - parsedSize, err := resource.ParseQuantity("10Gi") - require.NoError(t, err) - assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) -} - -func Test_reconcileHeaterVolumeUsingStorageSize(t *testing.T) { - ctx := context.TODO() - instance1 := newEmptyRpaasInstance() - instance1.Name = "instance-1" + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resources := []runtime.Object{} + scheme := newScheme() + corev1.AddToScheme(scheme) - resources := []runtime.Object{} - scheme := newScheme() - corev1.AddToScheme(scheme) + k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) + reconciler := &ReconcileRpaasInstance{ + client: k8sClient, + scheme: newScheme(), + } + err := reconciler.reconcileCacheHeaterVolume(ctx, rpaasInstance, &v1alpha1.RpaasPlan{Spec: tt.planSpec}) + require.NoError(t, err) - k8sClient := fake.NewFakeClientWithScheme(scheme, resources...) - reconciler := &ReconcileRpaasInstance{ - client: k8sClient, - scheme: newScheme(), - } + pvc := &corev1.PersistentVolumeClaim{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: rpaasInstance.Name + "-heater-volume", + Namespace: rpaasInstance.Namespace, + }, pvc) + require.NoError(t, err) - plan := &v1alpha1.RpaasPlan{ - Spec: v1alpha1.RpaasPlanSpec{ - Config: v1alpha1.NginxConfig{ - CacheSize: resourceMustParsePtr("10Gi"), - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ - StorageClassName: strPtr("my-storage-class"), - StorageSize: resourceMustParsePtr("100Gi"), - }, - }, - }, + tt.assert(t, pvc) + }) } - err := reconciler.reconcileCacheHeaterVolume(ctx, instance1, plan) - require.NoError(t, err) - - pvc := &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) - require.NoError(t, err) - - parsedSize, err := resource.ParseQuantity("100Gi") - require.NoError(t, err) - assert.Equal(t, parsedSize, pvc.Spec.Resources.Requests["storage"]) } func Test_destroyHeaterVolume(t *testing.T) { From a56f31db6fde8a419850bf0da807bb64b9fca3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 7 May 2020 11:13:01 -0300 Subject: [PATCH 42/47] Add header license --- pkg/apis/extensions/v1alpha1/rpaasinstance.go | 10 +++++++--- .../rpaasinstance/rpaasinstance_controller.go | 5 +---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/apis/extensions/v1alpha1/rpaasinstance.go b/pkg/apis/extensions/v1alpha1/rpaasinstance.go index f242fa5bb..a27ee10b9 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasinstance.go +++ b/pkg/apis/extensions/v1alpha1/rpaasinstance.go @@ -1,7 +1,11 @@ +// Copyright 2020 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package v1alpha1 -const( - teamOwnerLabel = "rpaas.extensions.tsuru.io/team-owner" +const ( + teamOwnerLabel = "rpaas.extensions.tsuru.io/team-owner" ) func (i *RpaasInstance) SetTeamOwner(team string) { @@ -12,7 +16,7 @@ func (i *RpaasInstance) SetTeamOwner(team string) { } func (i *RpaasInstance) TeamOwner() string { - return i.Labels[teamOwnerLabel] + return i.Labels[teamOwnerLabel] } func mergeMap(a, b map[string]string) map[string]string { diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index f402a0344..f102083d7 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -571,10 +571,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, } logrus.Infof("creating PersistentVolumeClaim %s", pvcName) - if err := r.client.Create(ctx, pvc); err != nil { - return err - } - return nil + return r.client.Create(ctx, pvc) } func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(ctx context.Context, instance *v1alpha1.RpaasInstance) error { From fee6d318a53504a319f9acc0601c0626bae73fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 7 May 2020 11:22:54 -0300 Subject: [PATCH 43/47] Add missing command to create nginx directory at init container --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index f102083d7..1660c712e 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -77,6 +77,7 @@ done "-c", ` mkdir -p ${CACHE_HEATER_MOUNTPOINT}/temp; +mkdir -p ${CACHE_HEATER_MOUNTPOINT}/nginx; mkdir -p ${CACHE_PATH}/rpaas/nginx_tmp; ${POD_CMD} `} From 41bf1ed455847d74bc20035e18962cdd5380be2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 11 May 2020 17:38:48 -0300 Subject: [PATCH 44/47] Fix paths of rsync --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 4 ++-- .../rpaasinstance/rpaasinstance_controller_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 1660c712e..9469b03ef 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -54,8 +54,8 @@ const ( cacheHeaterMountPoint = "/var/cache/cache-heater" - rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=${CACHE_HEATER_MOUNTPOINT}/temp ${CACHE_PATH}/rpaas/nginx ${CACHE_HEATER_MOUNTPOINT}" - rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=${CACHE_PATH}/rpaas/nginx_tmp ${CACHE_HEATER_MOUNTPOINT}/nginx ${CACHE_PATH}/rpaas" + rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=${CACHE_HEATER_MOUNTPOINT}/temp ${CACHE_PATH}/nginx ${CACHE_HEATER_MOUNTPOINT}" + rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=${CACHE_PATH}/nginx_tmp ${CACHE_HEATER_MOUNTPOINT}/nginx ${CACHE_PATH}" ) var ( diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index def231163..b4f671c47 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -1466,7 +1466,7 @@ func TestReconcile(t *testing.T) { CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ StorageClassName: strPtr("my-storage-class"), }, - CachePath: "/var/cache/nginx", + CachePath: "/var/cache/nginx/rpaas", CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ Schedule: "1 * * * *", Image: "test/test:latest", @@ -1520,13 +1520,13 @@ func TestReconcile(t *testing.T) { {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, {Name: "CACHE_HEATER_MOUNTPOINT", Value: "/var/cache/cache-heater"}, - {Name: "CACHE_PATH", Value: "/var/cache/nginx"}, + {Name: "CACHE_PATH", Value: "/var/cache/nginx/rpaas"}, {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-heater/nginx /var/cache/nginx/rpaas"}, }, initContainer.Env) assert.Equal(t, []corev1.VolumeMount{ {Name: "cache-heater-volume", MountPath: "/var/cache/cache-heater"}, - {Name: "cache-vol", MountPath: "/var/cache/nginx"}, + {Name: "cache-vol", MountPath: "/var/cache/nginx/rpaas"}, }, initContainer.VolumeMounts) cronJob := &batchv1beta1.CronJob{} @@ -1546,7 +1546,7 @@ func TestReconcile(t *testing.T) { {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, {Name: "CACHE_HEATER_MOUNTPOINT", Value: "/var/cache/cache-heater"}, - {Name: "CACHE_PATH", Value: "/var/cache/nginx"}, + {Name: "CACHE_PATH", Value: "/var/cache/nginx/rpaas"}, {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater"}, }, podSpec.Containers[0].Env) } From 29ee1f9ff4451f565c308d0b1ea74b679836b92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 12 May 2020 11:52:34 -0300 Subject: [PATCH 45/47] Add tags for each resource generated by rpaas instance --- .../rpaasinstance/rpaasinstance_controller.go | 44 +++++++++------ .../rpaasinstance_controller_test.go | 55 ++++++++++++++----- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 9469b03ef..548b9dd06 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -62,14 +62,15 @@ var ( defaultCacheHeaterCmdPodToPVC = []string{ "/bin/bash", "-c", - ` -pods=($(kubectl -n ${SERVICE_NAME} get pod -l rpaas.extensions.tsuru.io/service-name=${SERVICE_NAME} -l rpaas.extensions.tsuru.io/instance-name=${INSTANCE_NAME} --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); + `pods=($(kubectl -n ${SERVICE_NAME} get pod -l rpaas.extensions.tsuru.io/service-name=${SERVICE_NAME} -l rpaas.extensions.tsuru.io/instance-name=${INSTANCE_NAME} --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); for pod in ${pods[@]}; do kubectl -n ${SERVICE_NAME} exec ${pod} -- ${POD_CMD}; if [[ $? == 0 ]]; then exit 0; fi done +echo "No pods found"; +exit 1 `} defaultCacheHeaterCmdPVCToPod = []string{ @@ -238,7 +239,7 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile return reconcile.Result{}, err } - if err = r.reconcileHPA(ctx, *instance, *nginx); err != nil { + if err = r.reconcileHPA(ctx, instance, nginx); err != nil { return reconcile.Result{}, err } @@ -350,7 +351,7 @@ func (r *ReconcileRpaasInstance) listDefaultFlavors(ctx context.Context, instanc return result, nil } -func (r *ReconcileRpaasInstance) reconcileHPA(ctx context.Context, instance v1alpha1.RpaasInstance, nginx nginxv1alpha1.Nginx) error { +func (r *ReconcileRpaasInstance) reconcileHPA(ctx context.Context, instance *v1alpha1.RpaasInstance, nginx *nginxv1alpha1.Nginx) error { logger := log.WithName("reconcileHPA"). WithValues("RpaasInstance", types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}). WithValues("Nginx", types.NamespacedName{Name: nginx.Name, Namespace: nginx.Namespace}) @@ -524,7 +525,7 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, cacheHeaterStorage := plan.Spec.Config.CacheHeaterStorage volumeMode := corev1.PersistentVolumeFilesystem - labels := map[string]string{} + labels := labelsForRpaasInstance(instance) if teamOwner := instance.TeamOwner(); teamOwner != "" { labels[volumeTeamLabel] = teamOwner } @@ -688,14 +689,15 @@ func (r *ReconcileRpaasInstance) deleteOldConfig(ctx context.Context, instance * func newConfigMap(instance *v1alpha1.RpaasInstance, renderedTemplate string) *corev1.ConfigMap { hash := fmt.Sprintf("%x", sha256.Sum256([]byte(renderedTemplate))) + labels := labelsForRpaasInstance(instance) + labels["type"] = "config" + labels["instance"] = instance.Name + return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-config-%s", instance.Name, hash[:10]), Namespace: instance.Namespace, - Labels: map[string]string{ - "type": "config", - "instance": instance.Name, - }, + Labels: labels, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(instance, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, @@ -734,6 +736,7 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config Kind: "RpaasInstance", }), }, + Labels: labelsForRpaasInstance(instance), }, TypeMeta: metav1.TypeMeta{ Kind: "Nginx", @@ -805,7 +808,7 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config return n } -func newHPA(instance v1alpha1.RpaasInstance, nginx nginxv1alpha1.Nginx) autoscalingv2beta2.HorizontalPodAutoscaler { +func newHPA(instance *v1alpha1.RpaasInstance, nginx *nginxv1alpha1.Nginx) autoscalingv2beta2.HorizontalPodAutoscaler { var metrics []autoscalingv2beta2.MetricSpec if instance.Spec.Autoscale.TargetCPUUtilizationPercentage != nil { @@ -848,12 +851,13 @@ func newHPA(instance v1alpha1.RpaasInstance, nginx nginxv1alpha1.Nginx) autoscal Name: instance.Name, Namespace: instance.Namespace, OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(&instance, schema.GroupVersionKind{ + *metav1.NewControllerRef(instance, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, Version: v1alpha1.SchemeGroupVersion.Version, Kind: "RpaasInstance", }), }, + Labels: labelsForRpaasInstance(instance), }, Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ @@ -885,6 +889,10 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat if len(plan.Spec.Config.CacheHeaterSync.CmdPodToPVC) > 0 { cmds = plan.Spec.Config.CacheHeaterSync.CmdPodToPVC } + jobLabels := labelsForRpaasInstance(instance) + jobLabels["log-app-name"] = instance.Name + jobLabels["log-process-name"] = "cache-synchronize" + return &batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ Name: cronName, @@ -896,6 +904,7 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat Kind: "RpaasInstance", }), }, + Labels: labelsForRpaasInstance(instance), }, TypeMeta: metav1.TypeMeta{ APIVersion: "batch/v1beta1", @@ -905,14 +914,10 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat Schedule: schedule, ConcurrencyPolicy: batchv1beta1.ForbidConcurrent, JobTemplate: batchv1beta1.JobTemplateSpec{ - Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "log-app-name": instance.Name, - "log-process-name": "cache-synchronize", - }, + Labels: jobLabels, }, Spec: corev1.PodSpec{ ServiceAccountName: "rpaas-cache-heater-cronjob", @@ -1162,3 +1167,10 @@ func (r *ReconcileRpaasInstance) reconcilePorts(ctx context.Context, instance *e return instancePorts, nil } + +func labelsForRpaasInstance(instance *extensionsv1alpha1.RpaasInstance) map[string]string { + return map[string]string{ + "rpaas.extensions.tsuru.io/instance-name": instance.Name, + "rpaas.extensions.tsuru.io/plan-name": instance.Spec.PlanName, + } +} diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index b4f671c47..21213cba7 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -289,6 +289,7 @@ func TestReconcileRpaasInstance_getRpaasInstance(t *testing.T) { Namespace: instance1.Namespace, }, Spec: v1alpha1.RpaasInstanceSpec{ + PlanName: "my-plan", Service: &nginxv1alpha1.NginxService{ Annotations: map[string]string{ "default-service-annotation": "default", @@ -323,7 +324,8 @@ func TestReconcileRpaasInstance_getRpaasInstance(t *testing.T) { Namespace: instance2.Namespace, }, Spec: v1alpha1.RpaasInstanceSpec{ - Flavors: []string{"mint"}, + Flavors: []string{"mint"}, + PlanName: "my-plan", Lifecycle: &nginxv1alpha1.NginxLifecycle{ PostStart: &nginxv1alpha1.NginxLifecycleHandler{ Exec: &corev1.ExecAction{ @@ -378,7 +380,8 @@ func TestReconcileRpaasInstance_getRpaasInstance(t *testing.T) { Namespace: instance3.Namespace, }, Spec: v1alpha1.RpaasInstanceSpec{ - Flavors: []string{"mint", "mango"}, + Flavors: []string{"mint", "mango"}, + PlanName: "my-plan", Service: &nginxv1alpha1.NginxService{ Annotations: map[string]string{ "default-service-annotation": "default", @@ -430,6 +433,7 @@ func TestReconcileRpaasInstance_getRpaasInstance(t *testing.T) { }, }, Spec: v1alpha1.RpaasInstanceSpec{ + PlanName: "my-plan", Service: &nginxv1alpha1.NginxService{ Annotations: map[string]string{ "default-service-annotation": "default", @@ -531,14 +535,14 @@ func Test_reconcileHPA(t *testing.T) { tests := []struct { name string - instance v1alpha1.RpaasInstance - nginx nginxv1alpha1.Nginx + instance *v1alpha1.RpaasInstance + nginx *nginxv1alpha1.Nginx assertion func(t *testing.T, err error, got *autoscalingv2beta2.HorizontalPodAutoscaler) }{ { name: "when there is HPA resource but autoscale spec is nil", - instance: *instance2, - nginx: *nginx2, + instance: instance2, + nginx: nginx2, assertion: func(t *testing.T, err error, got *autoscalingv2beta2.HorizontalPodAutoscaler) { require.Error(t, err) assert.True(t, k8sErrors.IsNotFound(err)) @@ -546,8 +550,8 @@ func Test_reconcileHPA(t *testing.T) { }, { name: "when there is no HPA resource but autoscale spec is provided", - instance: *instance1, - nginx: *nginx1, + instance: instance1, + nginx: nginx1, assertion: func(t *testing.T, err error, got *autoscalingv2beta2.HorizontalPodAutoscaler) { require.NoError(t, err) require.NotNil(t, got) @@ -579,11 +583,16 @@ func Test_reconcileHPA(t *testing.T) { }, }, }, got.Spec.Metrics[1]) + + assert.Equal(t, map[string]string{ + "rpaas.extensions.tsuru.io/instance-name": "instance-1", + "rpaas.extensions.tsuru.io/plan-name": "my-plan", + }, got.ObjectMeta.Labels) }, }, { name: "when there is HPA resource but differs from autoscale spec", - instance: v1alpha1.RpaasInstance{ + instance: &v1alpha1.RpaasInstance{ TypeMeta: metav1.TypeMeta{ APIVersion: "extensions.tsuru.io/v1alpha1", Kind: "RpaasInstance", @@ -601,7 +610,7 @@ func Test_reconcileHPA(t *testing.T) { }, }, }, - nginx: *nginx2, + nginx: nginx2, assertion: func(t *testing.T, err error, got *autoscalingv2beta2.HorizontalPodAutoscaler) { require.NoError(t, err) require.NotNil(t, got) @@ -703,11 +712,13 @@ func Test_reconcileHeaterVolume(t *testing.T) { }, }, assert: func(t *testing.T, pvc *corev1.PersistentVolumeClaim) { - assert.Equal(t, 3, len(pvc.ObjectMeta.Labels)) + assert.Equal(t, 5, len(pvc.ObjectMeta.Labels)) assert.Equal(t, map[string]string{ - "some-label": "foo", - "other-label": "bar", - "tsuru.io/volume-team": "team-one", + "some-label": "foo", + "other-label": "bar", + "tsuru.io/volume-team": "team-one", + "rpaas.extensions.tsuru.io/instance-name": "my-instance", + "rpaas.extensions.tsuru.io/plan-name": "my-plan", }, pvc.ObjectMeta.Labels) }, }, @@ -829,6 +840,18 @@ func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { assert.Equal(t, "RpaasInstance", cronJob.ObjectMeta.OwnerReferences[0].Kind) assert.Equal(t, instance1.Name, cronJob.ObjectMeta.OwnerReferences[0].Name) + + assert.Equal(t, map[string]string{ + "rpaas.extensions.tsuru.io/instance-name": "instance-1", + "rpaas.extensions.tsuru.io/plan-name": "my-plan", + }, cronJob.ObjectMeta.Labels) + + assert.Equal(t, map[string]string{ + "log-app-name": "instance-1", + "log-process-name": "cache-synchronize", + "rpaas.extensions.tsuru.io/instance-name": "instance-1", + "rpaas.extensions.tsuru.io/plan-name": "my-plan", + }, cronJob.Spec.JobTemplate.Spec.Template.ObjectMeta.Labels) } func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { @@ -927,7 +950,9 @@ func newEmptyRpaasInstance() *v1alpha1.RpaasInstance { Name: "my-instance", Namespace: "default", }, - Spec: v1alpha1.RpaasInstanceSpec{}, + Spec: v1alpha1.RpaasInstanceSpec{ + PlanName: "my-plan", + }, } } From 38539323d3c1c7a954a7d058cc09b74ecdc7be08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 12 May 2020 16:35:58 -0300 Subject: [PATCH 46/47] Rename heater to snapshot --- deploy/role_binding.yaml | 4 +- deploy/service_account.yaml | 2 +- .../extensions/v1alpha1/rpaasplan_types.go | 10 +- .../v1alpha1/zz_generated.deepcopy.go | 20 +-- .../rpaasinstance/rpaasinstance_controller.go | 127 +++++++++--------- .../rpaasinstance_controller_test.go | 88 ++++++------ 6 files changed, 127 insertions(+), 124 deletions(-) diff --git a/deploy/role_binding.yaml b/deploy/role_binding.yaml index b1f522a57..e50499a7b 100644 --- a/deploy/role_binding.yaml +++ b/deploy/role_binding.yaml @@ -14,10 +14,10 @@ roleRef: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rpaas-cache-heater-cronjob + name: rpaas-cache-snapshot-cronjob subjects: - kind: ServiceAccount - name: rpaas-cache-heater + name: rpaas-cache-snapshot namespace: rpaas-operator-integration roleRef: kind: ClusterRole diff --git a/deploy/service_account.yaml b/deploy/service_account.yaml index a869a8b01..aa5c5f3fd 100644 --- a/deploy/service_account.yaml +++ b/deploy/service_account.yaml @@ -6,4 +6,4 @@ metadata: apiVersion: v1 kind: ServiceAccount metadata: - name: rpaas-cache-heater + name: rpaas-cache-snapshot diff --git a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go index b716f615e..e2f305b8d 100644 --- a/pkg/apis/extensions/v1alpha1/rpaasplan_types.go +++ b/pkg/apis/extensions/v1alpha1/rpaasplan_types.go @@ -67,9 +67,9 @@ type NginxConfig struct { CacheSize *resource.Quantity `json:"cacheSize,omitempty"` CacheZoneSize *resource.Quantity `json:"cacheZoneSize,omitempty"` - CacheHeaterEnabled bool `json:"cacheHeaterEnabled,omitempty"` - CacheHeaterStorage CacheHeaterStorage `json:"cacheHeaterStorage,omitempty"` - CacheHeaterSync CacheHeaterSyncSpec `json:"cacheHeaterSync,omitempty"` + CacheSnapshotEnabled bool `json:"cacheSnapshotEnabled,omitempty"` + CacheSnapshotStorage CacheSnapshotStorage `json:"cacheSnapshotStorage,omitempty"` + CacheSnapshotSync CacheSnapshotSyncSpec `json:"cacheSnapshotSync,omitempty"` HTTPListenOptions string `json:"httpListenOptions,omitempty"` HTTPSListenOptions string `json:"httpsListenOptions,omitempty"` @@ -86,7 +86,7 @@ type NginxConfig struct { WorkerConnections int `json:"workerConnections,omitempty"` } -type CacheHeaterSyncSpec struct { +type CacheSnapshotSyncSpec struct { // Schedule is the the cron time string format, see https://en.wikipedia.org/wiki/Cron. Schedule string `json:"schedule,omitempty"` @@ -101,7 +101,7 @@ type CacheHeaterSyncSpec struct { CmdPVCToPod []string `json:"cmdPVCToPod,omitempty"` } -type CacheHeaterStorage struct { +type CacheSnapshotStorage struct { StorageClassName *string `json:"storageClassName,omitempty"` StorageSize *resource.Quantity `json:"storageSize,omitempty"` VolumeLabels map[string]string `json:"volumeLabels,omitempty"` diff --git a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go index 7516daa3e..4a7700781 100644 --- a/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/extensions/v1alpha1/zz_generated.deepcopy.go @@ -48,7 +48,7 @@ func (in *Bind) DeepCopy() *Bind { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CacheHeaterStorage) DeepCopyInto(out *CacheHeaterStorage) { +func (in *CacheSnapshotStorage) DeepCopyInto(out *CacheSnapshotStorage) { *out = *in if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName @@ -70,18 +70,18 @@ func (in *CacheHeaterStorage) DeepCopyInto(out *CacheHeaterStorage) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheHeaterStorage. -func (in *CacheHeaterStorage) DeepCopy() *CacheHeaterStorage { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheSnapshotStorage. +func (in *CacheSnapshotStorage) DeepCopy() *CacheSnapshotStorage { if in == nil { return nil } - out := new(CacheHeaterStorage) + out := new(CacheSnapshotStorage) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CacheHeaterSyncSpec) DeepCopyInto(out *CacheHeaterSyncSpec) { +func (in *CacheSnapshotSyncSpec) DeepCopyInto(out *CacheSnapshotSyncSpec) { *out = *in if in.CmdPodToPVC != nil { in, out := &in.CmdPodToPVC, &out.CmdPodToPVC @@ -96,12 +96,12 @@ func (in *CacheHeaterSyncSpec) DeepCopyInto(out *CacheHeaterSyncSpec) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheHeaterSyncSpec. -func (in *CacheHeaterSyncSpec) DeepCopy() *CacheHeaterSyncSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheSnapshotSyncSpec. +func (in *CacheSnapshotSyncSpec) DeepCopy() *CacheSnapshotSyncSpec { if in == nil { return nil } - out := new(CacheHeaterSyncSpec) + out := new(CacheSnapshotSyncSpec) in.DeepCopyInto(out) return out } @@ -161,8 +161,8 @@ func (in *NginxConfig) DeepCopyInto(out *NginxConfig) { x := (*in).DeepCopy() *out = &x } - in.CacheHeaterStorage.DeepCopyInto(&out.CacheHeaterStorage) - in.CacheHeaterSync.DeepCopyInto(&out.CacheHeaterSync) + in.CacheSnapshotStorage.DeepCopyInto(&out.CacheSnapshotStorage) + in.CacheSnapshotSync.DeepCopyInto(&out.CacheSnapshotSync) if in.VTSEnabled != nil { in, out := &in.VTSEnabled, &out.VTSEnabled *out = new(bool) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 548b9dd06..3211352ea 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -44,22 +44,22 @@ import ( const ( defaultConfigHistoryLimit = 10 - defaultCacheHeaterCronImage = "bitnami/kubectl:latest" - defaultCacheHeaterSchedule = "* * * * *" + defaultCacheSnapshotCronImage = "bitnami/kubectl:latest" + defaultCacheSnapshotSchedule = "* * * * *" defaultPortAllocationResource = "default" volumeTeamLabel = "tsuru.io/volume-team" - cacheHeaterCronJobSuffix = "-heater-cron-job" - cacheHeaterVolumeSuffix = "-heater-volume" + cacheSnapshotCronJobSuffix = "-snapshot-cron-job" + cacheSnapshotVolumeSuffix = "-snapshot-volume" - cacheHeaterMountPoint = "/var/cache/cache-heater" + cacheSnapshotMountPoint = "/var/cache/cache-snapshot" - rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=${CACHE_HEATER_MOUNTPOINT}/temp ${CACHE_PATH}/nginx ${CACHE_HEATER_MOUNTPOINT}" - rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=${CACHE_PATH}/nginx_tmp ${CACHE_HEATER_MOUNTPOINT}/nginx ${CACHE_PATH}" + rsyncCommandPodToPVC = "rsync -avz --recursive --delete --temp-dir=${CACHE_SNAPSHOT_MOUNTPOINT}/temp ${CACHE_PATH}/nginx ${CACHE_SNAPSHOT_MOUNTPOINT}" + rsyncCommandPVCToPod = "rsync -avz --recursive --delete --temp-dir=${CACHE_PATH}/nginx_tmp ${CACHE_SNAPSHOT_MOUNTPOINT}/nginx ${CACHE_PATH}" ) var ( - defaultCacheHeaterCmdPodToPVC = []string{ + defaultCacheSnapshotCmdPodToPVC = []string{ "/bin/bash", "-c", `pods=($(kubectl -n ${SERVICE_NAME} get pod -l rpaas.extensions.tsuru.io/service-name=${SERVICE_NAME} -l rpaas.extensions.tsuru.io/instance-name=${INSTANCE_NAME} --field-selector status.phase=Running -o=jsonpath='{.items[*].metadata.name}')); @@ -73,12 +73,12 @@ echo "No pods found"; exit 1 `} - defaultCacheHeaterCmdPVCToPod = []string{ + defaultCacheSnapshotCmdPVCToPod = []string{ "/bin/bash", "-c", ` -mkdir -p ${CACHE_HEATER_MOUNTPOINT}/temp; -mkdir -p ${CACHE_HEATER_MOUNTPOINT}/nginx; +mkdir -p ${CACHE_SNAPSHOT_MOUNTPOINT}/temp; +mkdir -p ${CACHE_SNAPSHOT_MOUNTPOINT}/nginx; mkdir -p ${CACHE_PATH}/rpaas/nginx_tmp; ${POD_CMD} `} @@ -131,6 +131,9 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { IsController: true, OwnerType: &extensionsv1alpha1.RpaasInstance{}, }) + if err != nil { + return err + } err = c.Watch(&source.Kind{Type: &batchv1beta1.CronJob{}}, &handler.EnqueueRequestForOwner{ IsController: true, @@ -235,7 +238,7 @@ func (r *ReconcileRpaasInstance) Reconcile(request reconcile.Request) (reconcile return reconcile.Result{}, err } - if err = r.reconcileCacheHeater(ctx, instance, plan); err != nil { + if err = r.reconcileCacheSnapshot(ctx, instance, plan); err != nil { return reconcile.Result{}, err } @@ -459,25 +462,25 @@ func (r *ReconcileRpaasInstance) reconcileNginx(ctx context.Context, nginx *ngin return err } -func (r *ReconcileRpaasInstance) reconcileCacheHeater(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { - if plan.Spec.Config.CacheHeaterEnabled { - err := r.reconcileCacheHeaterCronJob(ctx, instance, plan) +func (r *ReconcileRpaasInstance) reconcileCacheSnapshot(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { + if plan.Spec.Config.CacheSnapshotEnabled { + err := r.reconcileCacheSnapshotCronJob(ctx, instance, plan) if err != nil { return err } - return r.reconcileCacheHeaterVolume(ctx, instance, plan) + return r.reconcileCacheSnapshotVolume(ctx, instance, plan) } - err := r.destroyCacheHeaterCronJob(ctx, instance) + err := r.destroyCacheSnapshotCronJob(ctx, instance) if err != nil { return err } - return r.destroyCacheHeaterVolume(ctx, instance) + return r.destroyCacheSnapshotVolume(ctx, instance) } -func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { +func (r *ReconcileRpaasInstance) reconcileCacheSnapshotCronJob(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { foundCronJob := &batchv1beta1.CronJob{} - cronName := instance.Name + cacheHeaterCronJobSuffix + cronName := instance.Name + cacheSnapshotCronJobSuffix err := r.client.Get(ctx, types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, foundCronJob) if err != nil && !k8sErrors.IsNotFound(err) { return err @@ -496,8 +499,8 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterCronJob(ctx context.Context return nil } -func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(ctx context.Context, instance *v1alpha1.RpaasInstance) error { - cronName := instance.Name + cacheHeaterCronJobSuffix +func (r *ReconcileRpaasInstance) destroyCacheSnapshotCronJob(ctx context.Context, instance *v1alpha1.RpaasInstance) error { + cronName := instance.Name + cacheSnapshotCronJobSuffix cronJob := &batchv1beta1.CronJob{} err := r.client.Get(ctx, types.NamespacedName{Name: cronName, Namespace: instance.Namespace}, cronJob) @@ -511,8 +514,8 @@ func (r *ReconcileRpaasInstance) destroyCacheHeaterCronJob(ctx context.Context, logrus.Infof("deleting cronjob %s", cronName) return r.client.Delete(ctx, cronJob) } -func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { - pvcName := instance.Name + cacheHeaterVolumeSuffix +func (r *ReconcileRpaasInstance) reconcileCacheSnapshotVolume(ctx context.Context, instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) error { + pvcName := instance.Name + cacheSnapshotVolumeSuffix pvc := &corev1.PersistentVolumeClaim{} err := r.client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) @@ -523,13 +526,13 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, return nil } - cacheHeaterStorage := plan.Spec.Config.CacheHeaterStorage + cacheSnapshotStorage := plan.Spec.Config.CacheSnapshotStorage volumeMode := corev1.PersistentVolumeFilesystem labels := labelsForRpaasInstance(instance) if teamOwner := instance.TeamOwner(); teamOwner != "" { labels[volumeTeamLabel] = teamOwner } - for k, v := range cacheHeaterStorage.VolumeLabels { + for k, v := range cacheSnapshotStorage.VolumeLabels { labels[k] = v } @@ -555,13 +558,13 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, corev1.ReadWriteMany, }, VolumeMode: &volumeMode, - StorageClassName: cacheHeaterStorage.StorageClassName, + StorageClassName: cacheSnapshotStorage.StorageClassName, }, } storageSize := plan.Spec.Config.CacheSize - if cacheHeaterStorage.StorageSize != nil && !cacheHeaterStorage.StorageSize.IsZero() { - storageSize = cacheHeaterStorage.StorageSize + if cacheSnapshotStorage.StorageSize != nil && !cacheSnapshotStorage.StorageSize.IsZero() { + storageSize = cacheSnapshotStorage.StorageSize } if storageSize != nil && !storageSize.IsZero() { @@ -576,8 +579,8 @@ func (r *ReconcileRpaasInstance) reconcileCacheHeaterVolume(ctx context.Context, return r.client.Create(ctx, pvc) } -func (r *ReconcileRpaasInstance) destroyCacheHeaterVolume(ctx context.Context, instance *v1alpha1.RpaasInstance) error { - pvcName := instance.Name + cacheHeaterVolumeSuffix +func (r *ReconcileRpaasInstance) destroyCacheSnapshotVolume(ctx context.Context, instance *v1alpha1.RpaasInstance) error { + pvcName := instance.Name + cacheSnapshotVolumeSuffix pvc := &corev1.PersistentVolumeClaim{} err := r.client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: instance.Namespace}, pvc) @@ -760,48 +763,48 @@ func newNginx(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan, config }, } - if !plan.Spec.Config.CacheHeaterEnabled { + if !plan.Spec.Config.CacheSnapshotEnabled { return n } - initCmd := defaultCacheHeaterCmdPVCToPod - if len(plan.Spec.Config.CacheHeaterSync.CmdPVCToPod) > 0 { - initCmd = plan.Spec.Config.CacheHeaterSync.CmdPVCToPod + initCmd := defaultCacheSnapshotCmdPVCToPod + if len(plan.Spec.Config.CacheSnapshotSync.CmdPVCToPod) > 0 { + initCmd = plan.Spec.Config.CacheSnapshotSync.CmdPVCToPod } n.Spec.PodTemplate.Volumes = append(n.Spec.PodTemplate.Volumes, corev1.Volume{ - Name: "cache-heater-volume", + Name: "cache-snapshot-volume", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Name + cacheHeaterVolumeSuffix, + ClaimName: instance.Name + cacheSnapshotVolumeSuffix, }, }, }) - cacheHeaterVolume := corev1.VolumeMount{ - Name: "cache-heater-volume", - MountPath: cacheHeaterMountPoint, + cacheSnapshotVolume := corev1.VolumeMount{ + Name: "cache-snapshot-volume", + MountPath: cacheSnapshotMountPoint, } - n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, cacheHeaterVolume) + n.Spec.PodTemplate.VolumeMounts = append(n.Spec.PodTemplate.VolumeMounts, cacheSnapshotVolume) n.Spec.PodTemplate.InitContainers = append(n.Spec.PodTemplate.InitContainers, corev1.Container{ - Name: "heat-cache", + Name: "restore-snapshot", Image: plan.Spec.Image, Command: []string{ initCmd[0], }, Args: initCmd[1:], VolumeMounts: []corev1.VolumeMount{ - cacheHeaterVolume, + cacheSnapshotVolume, { Name: "cache-vol", MountPath: plan.Spec.Config.CachePath, }, }, - Env: append(cacheHeaterEnvVars(instance, plan), corev1.EnvVar{ + Env: append(cacheSnapshotEnvVars(instance, plan), corev1.EnvVar{ Name: "POD_CMD", - Value: interpolateCacheHeaterPodCmdTemplate(rsyncCommandPVCToPod, plan), + Value: interpolateCacheSnapshotPodCmdTemplate(rsyncCommandPVCToPod, plan), }), }) @@ -873,21 +876,21 @@ func newHPA(instance *v1alpha1.RpaasInstance, nginx *nginxv1alpha1.Nginx) autosc } func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *batchv1beta1.CronJob { - cronName := instance.Name + cacheHeaterCronJobSuffix + cronName := instance.Name + cacheSnapshotCronJobSuffix - schedule := defaultCacheHeaterSchedule - if plan.Spec.Config.CacheHeaterSync.Schedule != "" { - schedule = plan.Spec.Config.CacheHeaterSync.Schedule + schedule := defaultCacheSnapshotSchedule + if plan.Spec.Config.CacheSnapshotSync.Schedule != "" { + schedule = plan.Spec.Config.CacheSnapshotSync.Schedule } - image := defaultCacheHeaterCronImage - if plan.Spec.Config.CacheHeaterSync.Image != "" { - image = plan.Spec.Config.CacheHeaterSync.Image + image := defaultCacheSnapshotCronImage + if plan.Spec.Config.CacheSnapshotSync.Image != "" { + image = plan.Spec.Config.CacheSnapshotSync.Image } - cmds := defaultCacheHeaterCmdPodToPVC - if len(plan.Spec.Config.CacheHeaterSync.CmdPodToPVC) > 0 { - cmds = plan.Spec.Config.CacheHeaterSync.CmdPodToPVC + cmds := defaultCacheSnapshotCmdPodToPVC + if len(plan.Spec.Config.CacheSnapshotSync.CmdPodToPVC) > 0 { + cmds = plan.Spec.Config.CacheSnapshotSync.CmdPodToPVC } jobLabels := labelsForRpaasInstance(instance) jobLabels["log-app-name"] = instance.Name @@ -920,7 +923,7 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat Labels: jobLabels, }, Spec: corev1.PodSpec{ - ServiceAccountName: "rpaas-cache-heater-cronjob", + ServiceAccountName: "rpaas-cache-snapshot-cronjob", Containers: []corev1.Container{ { Name: "cache-synchronize", @@ -929,9 +932,9 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat cmds[0], }, Args: cmds[1:], - Env: append(cacheHeaterEnvVars(instance, plan), corev1.EnvVar{ + Env: append(cacheSnapshotEnvVars(instance, plan), corev1.EnvVar{ Name: "POD_CMD", - Value: interpolateCacheHeaterPodCmdTemplate(rsyncCommandPodToPVC, plan), + Value: interpolateCacheSnapshotPodCmdTemplate(rsyncCommandPodToPVC, plan), }), }, }, @@ -944,19 +947,19 @@ func newCronJob(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) *bat } } -func interpolateCacheHeaterPodCmdTemplate(podCmd string, plan *v1alpha1.RpaasPlan) string { +func interpolateCacheSnapshotPodCmdTemplate(podCmd string, plan *v1alpha1.RpaasPlan) string { replacer := strings.NewReplacer( - "${CACHE_HEATER_MOUNTPOINT}", cacheHeaterMountPoint, + "${CACHE_SNAPSHOT_MOUNTPOINT}", cacheSnapshotMountPoint, "${CACHE_PATH}", plan.Spec.Config.CachePath, ) return replacer.Replace(podCmd) } -func cacheHeaterEnvVars(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) []corev1.EnvVar { +func cacheSnapshotEnvVars(instance *v1alpha1.RpaasInstance, plan *v1alpha1.RpaasPlan) []corev1.EnvVar { return []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: instance.Namespace}, {Name: "INSTANCE_NAME", Value: instance.Name}, - {Name: "CACHE_HEATER_MOUNTPOINT", Value: cacheHeaterMountPoint}, + {Name: "CACHE_SNAPSHOT_MOUNTPOINT", Value: cacheSnapshotMountPoint}, {Name: "CACHE_PATH", Value: plan.Spec.Config.CachePath}, } } diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go index 21213cba7..8c1bf6107 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller_test.go @@ -667,7 +667,7 @@ func Test_reconcileHPA(t *testing.T) { } } -func Test_reconcileHeaterVolume(t *testing.T) { +func Test_reconcileSnapshotVolume(t *testing.T) { ctx := context.TODO() rpaasInstance := newEmptyRpaasInstance() rpaasInstance.Name = "my-instance" @@ -683,7 +683,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { planSpec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ CacheSize: resourceMustParsePtr("10Gi"), - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + CacheSnapshotStorage: v1alpha1.CacheSnapshotStorage{ StorageClassName: strPtr("my-storage-class"), }, }, @@ -702,7 +702,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { name: "Should repass volume labels to PVC", planSpec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + CacheSnapshotStorage: v1alpha1.CacheSnapshotStorage{ StorageClassName: strPtr("my-storage-class"), VolumeLabels: map[string]string{ "some-label": "foo", @@ -714,9 +714,9 @@ func Test_reconcileHeaterVolume(t *testing.T) { assert: func(t *testing.T, pvc *corev1.PersistentVolumeClaim) { assert.Equal(t, 5, len(pvc.ObjectMeta.Labels)) assert.Equal(t, map[string]string{ - "some-label": "foo", - "other-label": "bar", - "tsuru.io/volume-team": "team-one", + "some-label": "foo", + "other-label": "bar", + "tsuru.io/volume-team": "team-one", "rpaas.extensions.tsuru.io/instance-name": "my-instance", "rpaas.extensions.tsuru.io/plan-name": "my-plan", }, pvc.ObjectMeta.Labels) @@ -727,7 +727,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { name: "Should priorize the team inside plan", planSpec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + CacheSnapshotStorage: v1alpha1.CacheSnapshotStorage{ VolumeLabels: map[string]string{ "tsuru.io/volume-team": "another-team", }, @@ -743,7 +743,7 @@ func Test_reconcileHeaterVolume(t *testing.T) { planSpec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ CacheSize: resourceMustParsePtr("10Gi"), - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + CacheSnapshotStorage: v1alpha1.CacheSnapshotStorage{ StorageSize: resourceMustParsePtr("100Gi"), }, }, @@ -766,12 +766,12 @@ func Test_reconcileHeaterVolume(t *testing.T) { client: k8sClient, scheme: newScheme(), } - err := reconciler.reconcileCacheHeaterVolume(ctx, rpaasInstance, &v1alpha1.RpaasPlan{Spec: tt.planSpec}) + err := reconciler.reconcileCacheSnapshotVolume(ctx, rpaasInstance, &v1alpha1.RpaasPlan{Spec: tt.planSpec}) require.NoError(t, err) pvc := &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: rpaasInstance.Name + "-heater-volume", + Name: rpaasInstance.Name + "-snapshot-volume", Namespace: rpaasInstance.Namespace, }, pvc) require.NoError(t, err) @@ -782,14 +782,14 @@ func Test_reconcileHeaterVolume(t *testing.T) { } -func Test_destroyHeaterVolume(t *testing.T) { +func Test_destroySnapshotVolume(t *testing.T) { ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: "instance-1-heater-volume", + Name: "instance-1-snapshot-volume", Namespace: "default", }, } @@ -803,15 +803,15 @@ func Test_destroyHeaterVolume(t *testing.T) { scheme: newScheme(), } - err := reconciler.destroyCacheHeaterVolume(ctx, instance1) + err := reconciler.destroyCacheSnapshotVolume(ctx, instance1) require.NoError(t, err) pvc = &corev1.PersistentVolumeClaim{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-volume", Namespace: instance1.Namespace}, pvc) + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-snapshot-volume", Namespace: instance1.Namespace}, pvc) require.True(t, k8sErrors.IsNotFound(err)) } -func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { +func Test_reconcileCacheSnapshotCronJobCreation(t *testing.T) { ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" @@ -831,11 +831,11 @@ func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{}, } - err := reconciler.reconcileCacheHeaterCronJob(ctx, instance1, plan) + err := reconciler.reconcileCacheSnapshotCronJob(ctx, instance1, plan) require.NoError(t, err) cronJob := &batchv1beta1.CronJob{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-cron-job", Namespace: instance1.Namespace}, cronJob) + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-snapshot-cron-job", Namespace: instance1.Namespace}, cronJob) require.NoError(t, err) assert.Equal(t, "RpaasInstance", cronJob.ObjectMeta.OwnerReferences[0].Kind) @@ -847,21 +847,21 @@ func Test_reconcileCacheHeaterCronJobCreation(t *testing.T) { }, cronJob.ObjectMeta.Labels) assert.Equal(t, map[string]string{ - "log-app-name": "instance-1", - "log-process-name": "cache-synchronize", + "log-app-name": "instance-1", + "log-process-name": "cache-synchronize", "rpaas.extensions.tsuru.io/instance-name": "instance-1", "rpaas.extensions.tsuru.io/plan-name": "my-plan", }, cronJob.Spec.JobTemplate.Spec.Template.ObjectMeta.Labels) } -func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { +func Test_reconcileCacheSnapshotCronJobUpdate(t *testing.T) { ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" previousCronJob := &batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: instance1.Name + "-heater-cronjob", + Name: instance1.Name + "-snapshot-cronjob", }, Spec: batchv1beta1.CronJobSpec{ Schedule: "old-schedule", @@ -882,18 +882,18 @@ func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { plan := &v1alpha1.RpaasPlan{ Spec: v1alpha1.RpaasPlanSpec{ Config: v1alpha1.NginxConfig{ - CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ + CacheSnapshotSync: v1alpha1.CacheSnapshotSyncSpec{ Schedule: "new-schedule", }, }, }, } - err := reconciler.reconcileCacheHeaterCronJob(ctx, instance1, plan) + err := reconciler.reconcileCacheSnapshotCronJob(ctx, instance1, plan) require.NoError(t, err) cronJob := &batchv1beta1.CronJob{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-cron-job", Namespace: instance1.Namespace}, cronJob) + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-snapshot-cron-job", Namespace: instance1.Namespace}, cronJob) require.NoError(t, err) assert.Equal(t, "RpaasInstance", cronJob.ObjectMeta.OwnerReferences[0].Kind) @@ -901,14 +901,14 @@ func Test_reconcileCacheHeaterCronJobUpdate(t *testing.T) { assert.Equal(t, "new-schedule", cronJob.Spec.Schedule) } -func Test_destroyHeaterCronJob(t *testing.T) { +func Test_destroySnapshotCronJob(t *testing.T) { ctx := context.TODO() instance1 := newEmptyRpaasInstance() instance1.Name = "instance-1" cronJob := &batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: instance1.Name + "-heater-cron-job", + Name: instance1.Name + "-snapshot-cron-job", Namespace: instance1.Namespace, }, } @@ -924,12 +924,12 @@ func Test_destroyHeaterCronJob(t *testing.T) { scheme: newScheme(), } - err := reconciler.destroyCacheHeaterCronJob(ctx, instance1) + err := reconciler.destroyCacheSnapshotCronJob(ctx, instance1) require.NoError(t, err) cronJob = &batchv1beta1.CronJob{} - err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-heater-cron-job", Namespace: instance1.Namespace}, cronJob) + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: instance1.Name + "-snapshot-cron-job", Namespace: instance1.Namespace}, cronJob) require.True(t, k8sErrors.IsNotFound(err)) } func int32Ptr(n int32) *int32 { @@ -1485,14 +1485,14 @@ func TestReconcile(t *testing.T) { Spec: v1alpha1.RpaasPlanSpec{ Image: "tsuru:mynginx:test", Config: v1alpha1.NginxConfig{ - CacheEnabled: v1alpha1.Bool(true), - CacheSize: resourceMustParsePtr("100M"), - CacheHeaterEnabled: true, - CacheHeaterStorage: v1alpha1.CacheHeaterStorage{ + CacheEnabled: v1alpha1.Bool(true), + CacheSize: resourceMustParsePtr("100M"), + CacheSnapshotEnabled: true, + CacheSnapshotStorage: v1alpha1.CacheSnapshotStorage{ StorageClassName: strPtr("my-storage-class"), }, CachePath: "/var/cache/nginx/rpaas", - CacheHeaterSync: v1alpha1.CacheHeaterSyncSpec{ + CacheSnapshotSync: v1alpha1.CacheSnapshotSyncSpec{ Schedule: "1 * * * *", Image: "test/test:latest", CmdPodToPVC: []string{ @@ -1528,15 +1528,15 @@ func TestReconcile(t *testing.T) { err = client.Get(context.TODO(), types.NamespacedName{Name: rpaas.Name, Namespace: rpaas.Namespace}, nginx) require.NoError(t, err) - assert.Equal(t, "cache-heater-volume", nginx.Spec.PodTemplate.Volumes[0].Name) - assert.Equal(t, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instance-heater-volume"}, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim) - assert.Equal(t, "cache-heater-volume", nginx.Spec.PodTemplate.VolumeMounts[0].Name) - assert.Equal(t, "/var/cache/cache-heater", nginx.Spec.PodTemplate.VolumeMounts[0].MountPath) + assert.Equal(t, "cache-snapshot-volume", nginx.Spec.PodTemplate.Volumes[0].Name) + assert.Equal(t, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "my-instance-snapshot-volume"}, nginx.Spec.PodTemplate.Volumes[0].PersistentVolumeClaim) + assert.Equal(t, "cache-snapshot-volume", nginx.Spec.PodTemplate.VolumeMounts[0].Name) + assert.Equal(t, "/var/cache/cache-snapshot", nginx.Spec.PodTemplate.VolumeMounts[0].MountPath) assert.Equal(t, resource.MustParse("100M"), *nginx.Spec.Cache.Size) initContainer := nginx.Spec.PodTemplate.InitContainers[0] - assert.Equal(t, "heat-cache", initContainer.Name) + assert.Equal(t, "restore-snapshot", initContainer.Name) assert.Equal(t, "tsuru:mynginx:test", initContainer.Image) assert.Equal(t, "/bin/bash", initContainer.Command[0]) assert.Equal(t, "-c", initContainer.Args[0]) @@ -1544,18 +1544,18 @@ func TestReconcile(t *testing.T) { assert.Equal(t, []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, - {Name: "CACHE_HEATER_MOUNTPOINT", Value: "/var/cache/cache-heater"}, + {Name: "CACHE_SNAPSHOT_MOUNTPOINT", Value: "/var/cache/cache-snapshot"}, {Name: "CACHE_PATH", Value: "/var/cache/nginx/rpaas"}, - {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-heater/nginx /var/cache/nginx/rpaas"}, + {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/nginx/rpaas/nginx_tmp /var/cache/cache-snapshot/nginx /var/cache/nginx/rpaas"}, }, initContainer.Env) assert.Equal(t, []corev1.VolumeMount{ - {Name: "cache-heater-volume", MountPath: "/var/cache/cache-heater"}, + {Name: "cache-snapshot-volume", MountPath: "/var/cache/cache-snapshot"}, {Name: "cache-vol", MountPath: "/var/cache/nginx/rpaas"}, }, initContainer.VolumeMounts) cronJob := &batchv1beta1.CronJob{} - err = client.Get(context.TODO(), types.NamespacedName{Name: "my-instance-heater-cron-job", Namespace: rpaas.Namespace}, cronJob) + err = client.Get(context.TODO(), types.NamespacedName{Name: "my-instance-snapshot-cron-job", Namespace: rpaas.Namespace}, cronJob) require.NoError(t, err) assert.Equal(t, "1 * * * *", cronJob.Spec.Schedule) @@ -1570,9 +1570,9 @@ func TestReconcile(t *testing.T) { assert.Equal(t, []corev1.EnvVar{ {Name: "SERVICE_NAME", Value: "default"}, {Name: "INSTANCE_NAME", Value: "my-instance"}, - {Name: "CACHE_HEATER_MOUNTPOINT", Value: "/var/cache/cache-heater"}, + {Name: "CACHE_SNAPSHOT_MOUNTPOINT", Value: "/var/cache/cache-snapshot"}, {Name: "CACHE_PATH", Value: "/var/cache/nginx/rpaas"}, - {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-heater/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-heater"}, + {Name: "POD_CMD", Value: "rsync -avz --recursive --delete --temp-dir=/var/cache/cache-snapshot/temp /var/cache/nginx/rpaas/nginx /var/cache/cache-snapshot"}, }, podSpec.Containers[0].Env) } From f9013379ae0bd5c0c3d9463f54f10d26954bdaca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 12 May 2020 18:00:37 -0300 Subject: [PATCH 47/47] Fix command to restore from snapshot --- pkg/controller/rpaasinstance/rpaasinstance_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/rpaasinstance/rpaasinstance_controller.go b/pkg/controller/rpaasinstance/rpaasinstance_controller.go index 3211352ea..2ab00c9b5 100644 --- a/pkg/controller/rpaasinstance/rpaasinstance_controller.go +++ b/pkg/controller/rpaasinstance/rpaasinstance_controller.go @@ -79,7 +79,7 @@ exit 1 ` mkdir -p ${CACHE_SNAPSHOT_MOUNTPOINT}/temp; mkdir -p ${CACHE_SNAPSHOT_MOUNTPOINT}/nginx; -mkdir -p ${CACHE_PATH}/rpaas/nginx_tmp; +mkdir -p ${CACHE_PATH}/nginx_tmp; ${POD_CMD} `} )