diff --git a/controllers/controller.go b/controllers/controller.go index b90fedd0c..1c2f385c6 100644 --- a/controllers/controller.go +++ b/controllers/controller.go @@ -18,6 +18,8 @@ import ( "strings" "text/template" + "github.com/jetstack/cert-manager/pkg/util/pki" + "github.com/imdario/mergo" "github.com/sirupsen/logrus" nginxv1alpha1 "github.com/tsuru/nginx-operator/api/v1alpha1" @@ -838,10 +840,16 @@ func (r *RpaasInstanceReconciler) renderTemplate(ctx context.Context, instance * return "", err } + fullCerts, err := r.getCertificates(ctx, instance.Namespace, instance.Spec.Certificates) + if err != nil { + return "", err + } + config := nginx.ConfigurationData{ - Instance: instance, - Config: &plan.Spec.Config, - Modules: make(map[string]interface{}), + Instance: instance, + Config: &plan.Spec.Config, + FullCertificates: fullCerts, + Modules: make(map[string]interface{}), } for _, mod := range modules { @@ -886,6 +894,47 @@ func (r *RpaasInstanceReconciler) getConfigurationBlocks(ctx context.Context, in return blocks, nil } +func (r *RpaasInstanceReconciler) getCertificates(ctx context.Context, namespace string, tlsSecret *nginxv1alpha1.TLSSecret) ([]nginx.CertificateData, error) { + if tlsSecret == nil || len(tlsSecret.Items) == 0 { + return nil, nil + } + cmName := types.NamespacedName{ + Name: tlsSecret.SecretName, + Namespace: namespace, + } + var secret corev1.Secret + if err := r.Client.Get(ctx, cmName, &secret); err != nil { + return nil, err + } + + var certsData []nginx.CertificateData + + for _, secretItem := range tlsSecret.Items { + if _, ok := secret.Data[secretItem.CertificateField]; !ok { + return nil, fmt.Errorf("certificate data not found") + } + if _, ok := secret.Data[secretItem.KeyField]; !ok { + return nil, fmt.Errorf("key data not found") + } + + certs, err := pki.DecodeX509CertificateChainBytes(secret.Data[secretItem.CertificateField]) + if err != nil { + return nil, err + } + + if len(certs) == 0 { + return nil, fmt.Errorf("no certificates found in pem file") + } + + certsData = append(certsData, nginx.CertificateData{ + Certificate: certs[0], + SecretItem: secretItem, + }) + } + + return certsData, nil +} + func (r *RpaasInstanceReconciler) updateLocationValues(ctx context.Context, instance *v1alpha1.RpaasInstance) error { for _, location := range instance.Spec.Locations { if location.Content == nil { diff --git a/go.sum b/go.sum index d5ae9fcd0..ac1d7fba9 100644 --- a/go.sum +++ b/go.sum @@ -1351,6 +1351,7 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-aggregator v0.21.0 h1:my2WYu8RJcj/ZzWAjPPnmxNRELk/iCdPjMaOmsZOeBU= k8s.io/kube-aggregator v0.21.0/go.mod h1:sIaa9L4QCBo9gjPyoGJns4cBjYVLq3s49FxF7m/1A0A= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= diff --git a/internal/pkg/rpaas/k8s.go b/internal/pkg/rpaas/k8s.go index 2ceb77e55..451e4368a 100644 --- a/internal/pkg/rpaas/k8s.go +++ b/internal/pkg/rpaas/k8s.go @@ -26,6 +26,7 @@ import ( jsonpatch "github.com/evanphx/json-patch/v5" "github.com/hashicorp/go-multierror" cmv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" + "github.com/jetstack/cert-manager/pkg/util/pki" "github.com/pkg/errors" "github.com/sirupsen/logrus" nginxv1alpha1 "github.com/tsuru/nginx-operator/api/v1alpha1" @@ -2005,16 +2006,17 @@ func (m *k8sRpaasManager) getCertificatesInfo(ctx context.Context, instance *v1a var certsInfo []clientTypes.CertificateInfo for _, cert := range certs { - c, err := tls.X509KeyPair([]byte(cert.Certificate), []byte(cert.Key)) + certs, err := pki.DecodeX509CertificateChainBytes([]byte(cert.Certificate)) if err != nil { return nil, err } - leaf, err := x509.ParseCertificate(c.Certificate[0]) - if err != nil { - return nil, err + if len(certs) == 0 { + return nil, fmt.Errorf("no certificates found in pem file") } + leaf := certs[0] + certsInfo = append(certsInfo, clientTypes.CertificateInfo{ Name: cert.Name, DNSNames: leaf.DNSNames, diff --git a/internal/pkg/rpaas/nginx/configuration_render.go b/internal/pkg/rpaas/nginx/configuration_render.go index fa68f7491..937d02624 100644 --- a/internal/pkg/rpaas/nginx/configuration_render.go +++ b/internal/pkg/rpaas/nginx/configuration_render.go @@ -6,6 +6,7 @@ package nginx import ( "bytes" + "crypto/x509" "fmt" "regexp" "strconv" @@ -13,6 +14,7 @@ import ( "text/template" sprig "github.com/Masterminds/sprig/v3" + nginxv1alpha1 "github.com/tsuru/nginx-operator/api/v1alpha1" "github.com/tsuru/rpaas-operator/api/v1alpha1" "github.com/tsuru/rpaas-operator/pkg/util" "k8s.io/apimachinery/pkg/api/resource" @@ -38,7 +40,13 @@ type ConfigurationData struct { Instance *v1alpha1.RpaasInstance // Modules is a map of installed modules, using a map instead of a slice // allow us to use `hasKey` inside templates. - Modules map[string]interface{} + Modules map[string]interface{} + FullCertificates []CertificateData +} + +type CertificateData struct { + Certificate *x509.Certificate + SecretItem nginxv1alpha1.TLSSecretItem } type rpaasConfigurationRenderer struct { @@ -193,6 +201,32 @@ func tlsSessionTicketTimeout(instance *v1alpha1.RpaasInstance) int { return nkeys * int(keyRotationInterval) } +type nginxServer struct { + Default bool + Certificate *x509.Certificate + SecretItem nginxv1alpha1.TLSSecretItem +} + +func nginxServers(c ConfigurationData) []nginxServer { + if len(c.FullCertificates) == 0 { + return []nginxServer{ + {Certificate: nil, Default: true}, + } + } + + servers := []nginxServer{} + + for i, cert := range c.FullCertificates { + servers = append(servers, nginxServer{ + Default: i == 0, + Certificate: cert.Certificate, + SecretItem: cert.SecretItem, + }) + } + + return servers +} + var internalTemplateFuncs = template.FuncMap(map[string]interface{}{ "boolValue": v1alpha1.BoolValue, "buildLocationKey": buildLocationKey, @@ -211,6 +245,7 @@ var internalTemplateFuncs = template.FuncMap(map[string]interface{}{ "tlsSessionTicketEnabled": tlsSessionTicketEnabled, "tlsSessionTicketKeys": tlsSessionTicketKeys, "tlsSessionTicketTimeout": tlsSessionTicketTimeout, + "nginxServers": nginxServers, "iterate": func(n int) []int { v := make([]int, n) for i := 0; i < n; i++ { @@ -242,6 +277,7 @@ var rawNginxConfiguration = ` {{- $config := .Config -}} {{- $instance := .Instance -}} {{- $modules := .Modules -}} +{{- $nginxServers := . | nginxServers -}} # This file was generated by RPaaS (https://github.com/tsuru/rpaas-operator.git) # Do not modify this file, any change will be lost. @@ -394,20 +430,21 @@ http { {{- end }} } + {{- range $_, $nginxServer := $nginxServers }} server { - listen {{ httpPort $instance }} default_server + listen {{ httpPort $instance }}{{ with $nginxServer.Default }} default_server{{ end }} {{- with $config.HTTPListenOptions }} {{ . }}{{ end }}; - {{- if $instance.Spec.Certificates }} - {{- range $_, $item := $instance.Spec.Certificates.Items }} - {{- if and (eq $item.CertificateField "default.crt") (eq $item.KeyField "default.key") }} - listen {{ httpsPort $instance }} default_server ssl http2 + {{- with $nginxServer.Certificate }} + listen {{ httpsPort $instance }}{{ with $nginxServer.Default }} default_server{{ end }} ssl http2 {{- with $config.HTTPSListenOptions }} {{ . }}{{ end }}; - ssl_certificate certs/{{ with $item.CertificatePath }}{{ . }}{{ else }}{{ $item.CertificateField }}{{ end }}; - ssl_certificate_key certs/{{ with $item.KeyPath }}{{ . }}{{ else }}{{ $item.KeyField }}{{ end }}; - {{- end }} + {{- with $nginxServer.Certificate.DNSNames}} + server_name {{- range $_, $dnsName := $nginxServer.Certificate.DNSNames }} {{ $dnsName }}{{- end }}; {{- end }} + + ssl_certificate certs/{{ with $nginxServer.SecretItem.CertificatePath }}{{ . }}{{ else }}{{ $nginxServer.SecretItem.CertificateField }}{{ end }}; + ssl_certificate_key certs/{{ with $nginxServer.SecretItem.KeyPath }}{{ . }}{{ else }}{{ $nginxServer.SecretItem.KeyField }}{{ end }}; {{- end }} {{- if boolValue $config.CacheEnabled }} @@ -469,5 +506,6 @@ http { {{ template "server" .}} } + {{- end}} } ` diff --git a/internal/pkg/rpaas/nginx/configuration_render_test.go b/internal/pkg/rpaas/nginx/configuration_render_test.go index c6ebb15e6..44d2d1da1 100644 --- a/internal/pkg/rpaas/nginx/configuration_render_test.go +++ b/internal/pkg/rpaas/nginx/configuration_render_test.go @@ -5,6 +5,7 @@ package nginx import ( + "crypto/x509" "testing" "github.com/stretchr/testify/assert" @@ -202,6 +203,17 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { }, }, }, + FullCertificates: []CertificateData{ + { + Certificate: &x509.Certificate{ + DNSNames: []string{"example.org"}, + }, + SecretItem: nginxv1alpha1.TLSSecretItem{ + CertificateField: "default.crt", + KeyField: "default.key", + }, + }, + }, }, assertion: func(t *testing.T, result string) { assert.Regexp(t, `listen 8443 default_server ssl http2;`, result) @@ -209,6 +221,62 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { assert.Regexp(t, `ssl_certificate_key certs/default.key;`, result) }, }, + { + name: "with many certs actived", + data: ConfigurationData{ + Config: &v1alpha1.NginxConfig{}, + Instance: &v1alpha1.RpaasInstance{ + Spec: v1alpha1.RpaasInstanceSpec{ + Certificates: &nginxv1alpha1.TLSSecret{ + SecretName: "secret-name", + Items: []nginxv1alpha1.TLSSecretItem{ + { + CertificateField: "example.crt", + KeyField: "example.key", + }, + { + CertificateField: "cert-manager.crt", + KeyField: "cert-manager.key", + }, + }, + }, + }, + }, + FullCertificates: []CertificateData{ + { + Certificate: &x509.Certificate{ + DNSNames: []string{"example.org", "example.io"}, + }, + SecretItem: nginxv1alpha1.TLSSecretItem{ + CertificateField: "example.crt", + KeyField: "example.key", + }, + }, + { + Certificate: &x509.Certificate{ + DNSNames: []string{"example.namespace.system.internal.company.com"}, + }, + SecretItem: nginxv1alpha1.TLSSecretItem{ + CertificateField: "cert-manager.crt", + KeyField: "cert-manager.key", + }, + }, + }, + }, + assertion: func(t *testing.T, result string) { + assert.Regexp(t, `listen 8443 default_server ssl http2;`, result) + assert.Regexp(t, `listen 8443 ssl http2;`, result) + + assert.Regexp(t, `ssl_certificate\s+certs/cert-manager.crt;`, result) + assert.Regexp(t, `ssl_certificate_key certs/cert-manager.key;`, result) + + assert.Regexp(t, `ssl_certificate\s+certs/example.crt;`, result) + assert.Regexp(t, `ssl_certificate_key certs/example.key;`, result) + + assert.Regexp(t, `server_name example.org example.io;`, result) + assert.Regexp(t, `server_name example.namespace.system.internal.company.com;`, result) + }, + }, { name: "with TLS actived and custom listen options", data: ConfigurationData{ @@ -230,6 +298,19 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { }, }, }, + FullCertificates: []CertificateData{ + { + Certificate: &x509.Certificate{ + DNSNames: []string{"example.org"}, + }, + SecretItem: nginxv1alpha1.TLSSecretItem{ + CertificateField: "default.crt", + CertificatePath: "custom_certificate_name.crt", + KeyField: "default.key", + KeyPath: "custom_key_name.key", + }, + }, + }, }, assertion: func(t *testing.T, result string) { assert.Regexp(t, `listen 8443 default_server ssl http2 backlog=2048 deferred reuseport;`, result) @@ -404,6 +485,19 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { }, }, }, + FullCertificates: []CertificateData{ + { + Certificate: &x509.Certificate{ + DNSNames: []string{"example.org"}, + }, + SecretItem: nginxv1alpha1.TLSSecretItem{ + CertificateField: "default.crt", + CertificatePath: "custom_certificate_name.crt", + KeyField: "default.key", + KeyPath: "custom_key_name.key", + }, + }, + }, }, assertion: func(t *testing.T, result string) { assert.Regexp(t, `listen 80 default_server;`, result) @@ -446,6 +540,19 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) { }, }, }, + FullCertificates: []CertificateData{ + { + Certificate: &x509.Certificate{ + DNSNames: []string{"example.org"}, + }, + SecretItem: nginxv1alpha1.TLSSecretItem{ + CertificateField: "default.crt", + CertificatePath: "custom_certificate_name.crt", + KeyField: "default.key", + KeyPath: "custom_key_name.key", + }, + }, + }, }, assertion: func(t *testing.T, result string) { assert.Regexp(t, `listen 20001 default_server;`, result)