Merge pull request #73 from tsuru/feat/cache-orchestration

Add support to cache snapshot

Makefile

@@ -5,6 +5,7 @@ KUBECONFIG ?= ~/.kube/config
 
 git_tag    := $(shell git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')
 git_commit := $(shell git rev-parse HEAD 2>/dev/null | cut -c1-7)
+go_root    := $(shell go env GOROOT)
 
 RPAAS_OPERATOR_VERSION ?= $(git_tag)/$(git_commit)
 GO_LDFLAGS ?= -X=github.com/tsuru/rpaas-operator/version.Version=$(RPAAS_OPERATOR_VERSION)
@@ -34,7 +35,7 @@ local: deploy/crds
 	operator-sdk up local --go-ldflags $(GO_LDFLAGS)
 
 generate:
-	operator-sdk generate k8s
+	GOROOT=$(go_root) operator-sdk generate k8s
 
 build: build/plugin/rpaasv2
 	operator-sdk build $(IMAGE_OPERATOR):$(TAG) --go-build-args "-ldflags $(GO_LDFLAGS)"

deploy/role.yaml

@@ -60,3 +60,27 @@ rules:
   - horizontalpodautoscalers
   verbs:
   - '*'
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  - cronjobs
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rpaas-cache-syncer
+rules:
+- apiGroups: [""]
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+- apiGroups: [""]
+  resources:
+  - pods/exec
+  verbs:
+  - create

deploy/role_binding.yaml

@@ -10,3 +10,16 @@ roleRef:
   kind: ClusterRole
   name: rpaas-operator
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rpaas-cache-snapshot-cronjob
+subjects:
+- kind: ServiceAccount
+  name: rpaas-cache-snapshot
+  namespace: rpaas-operator-integration
+roleRef:
+  kind: ClusterRole
+  name: rpaas-cache-syncer
+  apiGroup: rbac.authorization.k8s.io

deploy/service_account.yaml

@@ -2,3 +2,8 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: rpaas-operator
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rpaas-cache-snapshot

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.4.0
-	github.com/tsuru/nginx-operator v0.5.0
+	github.com/tsuru/nginx-operator v0.5.2
 	github.com/urfave/cli/v2 v2.0.0
 	github.com/willf/bitset v1.1.10
 	k8s.io/api v0.0.0

go.sum

@@ -611,8 +611,8 @@ github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1C
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tsuru/config v0.0.0-20180418191556-87403ee7da02/go.mod h1:bilf/jr5AGl9raMFnhM9hsp+ms6PK752UF89ODoRHW4=
-github.com/tsuru/nginx-operator v0.5.0 h1:oWSu/9WHm+wKbvJSZn0NebQZCw2R0R5HYgGNZcasp+w=
-github.com/tsuru/nginx-operator v0.5.0/go.mod h1:hvXj+bM5dUsnOt/PQE1ktHnLxM1ucnhKM0zOdHZyzsg=
+github.com/tsuru/nginx-operator v0.5.2 h1:5i+0EKOnvVrkXEiXIxzKT75h4y80dj50kiqZwqlyeUk=
+github.com/tsuru/nginx-operator v0.5.2/go.mod h1:hvXj+bM5dUsnOt/PQE1ktHnLxM1ucnhKM0zOdHZyzsg=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg=

internal/pkg/rpaas/k8s.go

@@ -112,7 +112,7 @@ func (m *k8sRpaasManager) CreateInstance(ctx context.Context, args CreateArgs) e
 	}
 
 	setDescription(instance, args.Description)
-	setTeamOwner(instance, args.Team)
+	instance.SetTeamOwner(args.Team)
 	setTags(instance, args.Tags)
 	setIP(instance, args.IP())
 
@@ -145,7 +145,7 @@ func (m *k8sRpaasManager) UpdateInstance(ctx context.Context, instanceName strin
 	instance.Spec.Flavors = args.Flavors()
 
 	setDescription(instance, args.Description)
-	setTeamOwner(instance, args.Team)
+	instance.SetTeamOwner(args.Team)
 	setTags(instance, args.Tags)
 	setIP(instance, args.IP())
 
@@ -1411,18 +1411,6 @@ func setTags(instance *v1alpha1.RpaasInstance, tags []string) {
 	})
 }
 
-func setTeamOwner(instance *v1alpha1.RpaasInstance, team string) {
-	if instance == nil {
-		return
-	}
-
-	newLabels := map[string]string{labelKey("team-owner"): team}
-
-	instance.Annotations = mergeMap(instance.Annotations, newLabels)
-	instance.Labels = mergeMap(instance.Labels, newLabels)
-	instance.Spec.PodTemplate.Labels = mergeMap(instance.Spec.PodTemplate.Labels, newLabels)
-}
-
 func (m *k8sRpaasManager) GetInstanceInfo(ctx context.Context, instanceName string) (*clientTypes.InstanceInfo, error) {
 	instance, err := m.GetInstance(ctx, instanceName)
 	if err != nil {

internal/pkg/rpaas/nginx/configuration_render.go

@@ -7,11 +7,13 @@ package nginx
 import (
 	"bytes"
 	"fmt"
+	"strconv"
 	"strings"
 	"text/template"
 
 	"github.com/tsuru/rpaas-operator/pkg/apis/extensions/v1alpha1"
 	"github.com/tsuru/rpaas-operator/pkg/util"
+	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 type ConfigurationRenderer interface {
@@ -145,6 +147,15 @@ func managePort(instance *v1alpha1.RpaasInstance) int32 {
 	return defaultManagePort
 }
 
+func k8sQuantityToNginx(quantity *resource.Quantity) string {
+	if quantity == nil || quantity.IsZero() {
+		return "0"
+	}
+
+	bytesN, _ := quantity.AsInt64()
+	return strconv.Itoa(int(bytesN))
+}
+
 var templateFuncs = template.FuncMap(map[string]interface{}{
 	"boolValue":          v1alpha1.BoolValue,
 	"buildLocationKey":   buildLocationKey,
@@ -159,6 +170,7 @@ var templateFuncs = template.FuncMap(map[string]interface{}{
 	"contains":           strings.Contains,
 	"hasPrefix":          strings.HasPrefix,
 	"hasSuffix":          strings.HasSuffix,
+	"k8sQuantityToNginx": k8sQuantityToNginx,
 })
 
 var defaultMainTemplate = template.Must(template.New("main").
@@ -217,9 +229,9 @@ http {
     proxy_http_version 1.1;
 
     {{- if boolValue $config.CacheEnabled }}
-    proxy_cache_path {{ $config.CachePath }}/nginx levels=1:2 keys_zone=rpaas:{{ $config.CacheZoneSize }}
+    proxy_cache_path {{ $config.CachePath }}/nginx levels=1:2 keys_zone=rpaas:{{ k8sQuantityToNginx $config.CacheZoneSize }}
         {{- with $config.CacheInactive }} inactive={{ . }}{{ end }}
-        {{- with $config.CacheSize }} max_size={{ . }}{{ end }}
+        {{- with $config.CacheSize }} max_size={{ k8sQuantityToNginx . }}{{ end }}
         {{- with $config.CacheLoaderFiles }} loader_files={{ . }}{{ end }};
 
     proxy_temp_path {{ $config.CachePath }}/nginx_tmp 1 2;
@@ -311,6 +323,10 @@ http {
         {{- end }}
         {{- end }}
 
+        {{- if boolValue $config.CacheEnabled }}
+        proxy_cache rpaas;
+        {{- end }}
+
         location = /_nginx_healthcheck {
             {{- if boolValue $config.VTSEnabled }}
             vhost_traffic_status_bypass_limit on;

internal/pkg/rpaas/nginx/configuration_render_test.go

@@ -12,9 +12,13 @@ import (
 	nginxv1alpha1 "github.com/tsuru/nginx-operator/pkg/apis/nginx/v1alpha1"
 	"github.com/tsuru/rpaas-operator/pkg/apis/extensions/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 func TestRpaasConfigurationRenderer_Render(t *testing.T) {
+	size100MB := resource.MustParse("100Mi")
+	size300MB := resource.MustParse("300Mi")
+
 	tests := []struct {
 		name      string
 		blocks    ConfigurationBlocks
@@ -70,19 +74,21 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) {
 				Config: &v1alpha1.NginxConfig{
 					CacheEnabled:  v1alpha1.Bool(true),
 					CachePath:     "/path/to/cache/dir",
-					CacheZoneSize: "100m",
+					CacheZoneSize: &size100MB,
 				},
 				Instance: &v1alpha1.RpaasInstance{},
 			},
 			assertion: func(t *testing.T, result string) {
-				assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100m;`, result)
+				assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:104857600;`, result)
 				assert.Regexp(t, `proxy_temp_path /path/to/cache/dir/nginx_tmp 1 2;`, result)
 				assert.Regexp(t, `server {
 \s+listen 8800;
 \s+location ~ \^/purge/\(\.\+\) {
 \s+proxy_cache_purge rpaas \$1\$is_args\$args;
 \s+}
 \s+}`, result)
+				assert.Regexp(t, `proxy_cache rpaas;`, result)
+
 			},
 		},
 		{
@@ -93,13 +99,13 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) {
 					CachePath:        "/path/to/cache/dir",
 					CacheInactive:    "12h",
 					CacheLoaderFiles: 1000,
-					CacheSize:        "300m",
-					CacheZoneSize:    "100m",
+					CacheSize:        &size300MB,
+					CacheZoneSize:    &size100MB,
 				},
 				Instance: &v1alpha1.RpaasInstance{},
 			},
 			assertion: func(t *testing.T, result string) {
-				assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:100m inactive=12h max_size=300m loader_files=1000;`, result)
+				assert.Regexp(t, `proxy_cache_path /path/to/cache/dir/nginx levels=1:2 keys_zone=rpaas:104857600 inactive=12h max_size=314572800 loader_files=1000;`, result)
 				assert.Regexp(t, `proxy_temp_path /path/to/cache/dir/nginx_tmp 1 2;`, result)
 				assert.Regexp(t, `server {
 \s+listen 8800;
@@ -556,3 +562,26 @@ func Test_hasRootPath(t *testing.T) {
 		})
 	}
 }
+
+func TestK8sQuantityToNginx(t *testing.T) {
+	type expectation struct {
+		k8sQuantity   string
+		nginxQuantity string
+	}
+
+	expectations := []expectation{
+		{"100Ki", "102400"},
+		{"100Mi", "104857600"},
+		{"100M", "100000000"},
+		{"1Gi", "1073741824"},
+		{"1G", "1000000000"},
+		{"1024Gi", "1099511627776"},
+		{"2Ti", "2199023255552"},
+	}
+
+	for _, expectation := range expectations {
+		k8sQuantity := resource.MustParse(expectation.k8sQuantity)
+		nginxQuantity := k8sQuantityToNginx(&k8sQuantity)
+		assert.Equal(t, expectation.nginxQuantity, nginxQuantity)
+	}
+}

pkg/apis/extensions/v1alpha1/rpaasinstance.go

@@ -0,0 +1,30 @@
+// Copyright 2020 tsuru authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package v1alpha1
+
+const (
+	teamOwnerLabel = "rpaas.extensions.tsuru.io/team-owner"
+)
+
+func (i *RpaasInstance) SetTeamOwner(team string) {
+	newLabels := map[string]string{teamOwnerLabel: team}
+	i.Labels = mergeMap(i.Labels, newLabels)
+	i.Annotations = mergeMap(i.Annotations, newLabels)
+	i.Spec.PodTemplate.Labels = mergeMap(i.Spec.PodTemplate.Labels, newLabels)
+}
+
+func (i *RpaasInstance) TeamOwner() string {
+	return i.Labels[teamOwnerLabel]
+}
+
+func mergeMap(a, b map[string]string) map[string]string {
+	if a == nil {
+		return b
+	}
+	for k, v := range b {
+		a[k] = v
+	}
+	return a
+}

pkg/apis/extensions/v1alpha1/rpaasinstance_test.go

@@ -0,0 +1,35 @@
+// Copyright 2020 tsuru authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package v1alpha1
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_SetTeamOwner(t *testing.T) {
+	instance := &RpaasInstance{}
+	instance.SetTeamOwner("team-one")
+	expected := map[string]string{teamOwnerLabel: "team-one"}
+	assert.Equal(t, expected, instance.Labels)
+	assert.Equal(t, expected, instance.Annotations)
+	assert.Equal(t, expected, instance.Spec.PodTemplate.Labels)
+
+	instance.SetTeamOwner("team-two")
+	expected = map[string]string{teamOwnerLabel: "team-two"}
+	assert.Equal(t, expected, instance.Labels)
+	assert.Equal(t, expected, instance.Annotations)
+	assert.Equal(t, expected, instance.Spec.PodTemplate.Labels)
+}
+
+func Test_GetTeamOwner(t *testing.T) {
+	instance := &RpaasInstance{}
+	owner := instance.TeamOwner()
+	assert.Equal(t, "", owner)
+	instance.SetTeamOwner("team-one")
+	owner = instance.TeamOwner()
+	assert.Equal(t, "team-one", owner)
+}

pkg/apis/extensions/v1alpha1/rpaasplan_types.go

@@ -6,6 +6,7 @@ package v1alpha1
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -59,12 +60,16 @@ type NginxConfig struct {
 
 	UpstreamKeepalive int `json:"upstreamKeepalive,omitempty"`
 
-	CacheEnabled     *bool  `json:"cacheEnabled,omitempty"`
-	CacheInactive    string `json:"cacheInactive,omitempty"`
-	CacheLoaderFiles int    `json:"cacheLoaderFiles,omitempty"`
-	CachePath        string `json:"cachePath,omitempty"`
-	CacheSize        string `json:"cacheSize,omitempty"`
-	CacheZoneSize    string `json:"cacheZoneSize,omitempty"`
+	CacheEnabled     *bool              `json:"cacheEnabled,omitempty"`
+	CacheInactive    string             `json:"cacheInactive,omitempty"`
+	CacheLoaderFiles int                `json:"cacheLoaderFiles,omitempty"`
+	CachePath        string             `json:"cachePath,omitempty"`
+	CacheSize        *resource.Quantity `json:"cacheSize,omitempty"`
+	CacheZoneSize    *resource.Quantity `json:"cacheZoneSize,omitempty"`
+
+	CacheSnapshotEnabled bool                  `json:"cacheSnapshotEnabled,omitempty"`
+	CacheSnapshotStorage CacheSnapshotStorage  `json:"cacheSnapshotStorage,omitempty"`
+	CacheSnapshotSync    CacheSnapshotSyncSpec `json:"cacheSnapshotSync,omitempty"`
 
 	HTTPListenOptions  string `json:"httpListenOptions,omitempty"`
 	HTTPSListenOptions string `json:"httpsListenOptions,omitempty"`
@@ -81,6 +86,27 @@ type NginxConfig struct {
 	WorkerConnections int `json:"workerConnections,omitempty"`
 }
 
+type CacheSnapshotSyncSpec struct {
+	// Schedule is the the cron time string format, see https://en.wikipedia.org/wiki/Cron.
+	Schedule string `json:"schedule,omitempty"`
+
+	// Container is the image used to sync the containers
+	// default is bitnami/kubectl:latest
+	Image string `json:"image,omitempty"`
+
+	// CmdPodToPVC is used to customize command used to sync memory cache (POD) to persistent storage (PVC)
+	CmdPodToPVC []string `json:"cmdPodToPVC,omitempty"`
+
+	// CmdPVCToPod is used to customize command used to sync persistent storage (PVC) to memory cache (POD)
+	CmdPVCToPod []string `json:"cmdPVCToPod,omitempty"`
+}
+
+type CacheSnapshotStorage struct {
+	StorageClassName *string            `json:"storageClassName,omitempty"`
+	StorageSize      *resource.Quantity `json:"storageSize,omitempty"`
+	VolumeLabels     map[string]string  `json:"volumeLabels,omitempty"`
+}
+
 func Bool(v bool) *bool {
 	return &v
 }