Skip to content

Commit

Permalink
Provide nginx with many server blocks for each certificate (#105)
Browse files Browse the repository at this point in the history 
* First draft to provide nginx with many server blocks

* Handle default_server at first value

* read certificate without touch on private key
  • Loading branch information
@wpjunior
wpjunior authored Aug 25, 2021
1 parent 5bc8423 commit 8c46831
Show file tree
Hide file tree
Showing 5 changed files with 213 additions and 16 deletions.
55 changes: 52 additions & 3 deletions controllers/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"strings"
"text/template"

"github.com/jetstack/cert-manager/pkg/util/pki"

"github.com/imdario/mergo"
"github.com/sirupsen/logrus"
nginxv1alpha1 "github.com/tsuru/nginx-operator/api/v1alpha1"
Expand Down Expand Up @@ -838,10 +840,16 @@ func (r *RpaasInstanceReconciler) renderTemplate(ctx context.Context, instance *
return "", err
}

fullCerts, err := r.getCertificates(ctx, instance.Namespace, instance.Spec.Certificates)
if err != nil {
return "", err
}

config := nginx.ConfigurationData{
Instance: instance,
Config: &plan.Spec.Config,
Modules: make(map[string]interface{}),
Instance: instance,
Config: &plan.Spec.Config,
FullCertificates: fullCerts,
Modules: make(map[string]interface{}),
}

for _, mod := range modules {
Expand Down Expand Up @@ -886,6 +894,47 @@ func (r *RpaasInstanceReconciler) getConfigurationBlocks(ctx context.Context, in
return blocks, nil
}

func (r *RpaasInstanceReconciler) getCertificates(ctx context.Context, namespace string, tlsSecret *nginxv1alpha1.TLSSecret) ([]nginx.CertificateData, error) {
if tlsSecret == nil || len(tlsSecret.Items) == 0 {
return nil, nil
}
cmName := types.NamespacedName{
Name: tlsSecret.SecretName,
Namespace: namespace,
}
var secret corev1.Secret
if err := r.Client.Get(ctx, cmName, &secret); err != nil {
return nil, err
}

var certsData []nginx.CertificateData

for _, secretItem := range tlsSecret.Items {
if _, ok := secret.Data[secretItem.CertificateField]; !ok {
return nil, fmt.Errorf("certificate data not found")
}
if _, ok := secret.Data[secretItem.KeyField]; !ok {
return nil, fmt.Errorf("key data not found")
}

certs, err := pki.DecodeX509CertificateChainBytes(secret.Data[secretItem.CertificateField])
if err != nil {
return nil, err
}

if len(certs) == 0 {
return nil, fmt.Errorf("no certificates found in pem file")
}

certsData = append(certsData, nginx.CertificateData{
Certificate: certs[0],
SecretItem: secretItem,
})
}

return certsData, nil
}

func (r *RpaasInstanceReconciler) updateLocationValues(ctx context.Context, instance *v1alpha1.RpaasInstance) error {
for _, location := range instance.Spec.Locations {
if location.Content == nil {
Expand Down
1 change: 1 addition & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1351,6 +1351,7 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-aggregator v0.21.0 h1:my2WYu8RJcj/ZzWAjPPnmxNRELk/iCdPjMaOmsZOeBU=
k8s.io/kube-aggregator v0.21.0/go.mod h1:sIaa9L4QCBo9gjPyoGJns4cBjYVLq3s49FxF7m/1A0A=
k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
Expand Down
10 changes: 6 additions & 4 deletions internal/pkg/rpaas/k8s.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
jsonpatch "github.com/evanphx/json-patch/v5"
"github.com/hashicorp/go-multierror"
cmv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
"github.com/jetstack/cert-manager/pkg/util/pki"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
nginxv1alpha1 "github.com/tsuru/nginx-operator/api/v1alpha1"
Expand Down Expand Up @@ -2005,16 +2006,17 @@ func (m *k8sRpaasManager) getCertificatesInfo(ctx context.Context, instance *v1a

var certsInfo []clientTypes.CertificateInfo
for _, cert := range certs {
c, err := tls.X509KeyPair([]byte(cert.Certificate), []byte(cert.Key))
certs, err := pki.DecodeX509CertificateChainBytes([]byte(cert.Certificate))
if err != nil {
return nil, err
}

leaf, err := x509.ParseCertificate(c.Certificate[0])
if err != nil {
return nil, err
if len(certs) == 0 {
return nil, fmt.Errorf("no certificates found in pem file")
}

leaf := certs[0]

certsInfo = append(certsInfo, clientTypes.CertificateInfo{
Name: cert.Name,
DNSNames: leaf.DNSNames,
Expand Down
56 changes: 47 additions & 9 deletions internal/pkg/rpaas/nginx/configuration_render.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ package nginx

import (
"bytes"
"crypto/x509"
"fmt"
"regexp"
"strconv"
"strings"
"text/template"

sprig "github.com/Masterminds/sprig/v3"
nginxv1alpha1 "github.com/tsuru/nginx-operator/api/v1alpha1"
"github.com/tsuru/rpaas-operator/api/v1alpha1"
"github.com/tsuru/rpaas-operator/pkg/util"
"k8s.io/apimachinery/pkg/api/resource"
Expand All @@ -38,7 +40,13 @@ type ConfigurationData struct {
Instance *v1alpha1.RpaasInstance
// Modules is a map of installed modules, using a map instead of a slice
// allow us to use `hasKey` inside templates.
Modules map[string]interface{}
Modules map[string]interface{}
FullCertificates []CertificateData
}

type CertificateData struct {
Certificate *x509.Certificate
SecretItem nginxv1alpha1.TLSSecretItem
}

type rpaasConfigurationRenderer struct {
Expand Down Expand Up @@ -193,6 +201,32 @@ func tlsSessionTicketTimeout(instance *v1alpha1.RpaasInstance) int {
return nkeys * int(keyRotationInterval)
}

type nginxServer struct {
Default bool
Certificate *x509.Certificate
SecretItem nginxv1alpha1.TLSSecretItem
}

func nginxServers(c ConfigurationData) []nginxServer {
if len(c.FullCertificates) == 0 {
return []nginxServer{
{Certificate: nil, Default: true},
}
}

servers := []nginxServer{}

for i, cert := range c.FullCertificates {
servers = append(servers, nginxServer{
Default: i == 0,
Certificate: cert.Certificate,
SecretItem: cert.SecretItem,
})
}

return servers
}

var internalTemplateFuncs = template.FuncMap(map[string]interface{}{
"boolValue": v1alpha1.BoolValue,
"buildLocationKey": buildLocationKey,
Expand All @@ -211,6 +245,7 @@ var internalTemplateFuncs = template.FuncMap(map[string]interface{}{
"tlsSessionTicketEnabled": tlsSessionTicketEnabled,
"tlsSessionTicketKeys": tlsSessionTicketKeys,
"tlsSessionTicketTimeout": tlsSessionTicketTimeout,
"nginxServers": nginxServers,
"iterate": func(n int) []int {
v := make([]int, n)
for i := 0; i < n; i++ {
Expand Down Expand Up @@ -242,6 +277,7 @@ var rawNginxConfiguration = `
{{- $config := .Config -}}
{{- $instance := .Instance -}}
{{- $modules := .Modules -}}
{{- $nginxServers := . | nginxServers -}}
# This file was generated by RPaaS (https://github.com/tsuru/rpaas-operator.git)
# Do not modify this file, any change will be lost.
Expand Down Expand Up @@ -394,20 +430,21 @@ http {
{{- end }}
}
{{- range $_, $nginxServer := $nginxServers }}
server {
listen {{ httpPort $instance }} default_server
listen {{ httpPort $instance }}{{ with $nginxServer.Default }} default_server{{ end }}
{{- with $config.HTTPListenOptions }} {{ . }}{{ end }};
{{- if $instance.Spec.Certificates }}
{{- range $_, $item := $instance.Spec.Certificates.Items }}
{{- if and (eq $item.CertificateField "default.crt") (eq $item.KeyField "default.key") }}
listen {{ httpsPort $instance }} default_server ssl http2
{{- with $nginxServer.Certificate }}
listen {{ httpsPort $instance }}{{ with $nginxServer.Default }} default_server{{ end }} ssl http2
{{- with $config.HTTPSListenOptions }} {{ . }}{{ end }};
ssl_certificate certs/{{ with $item.CertificatePath }}{{ . }}{{ else }}{{ $item.CertificateField }}{{ end }};
ssl_certificate_key certs/{{ with $item.KeyPath }}{{ . }}{{ else }}{{ $item.KeyField }}{{ end }};
{{- end }}
{{- with $nginxServer.Certificate.DNSNames}}
server_name {{- range $_, $dnsName := $nginxServer.Certificate.DNSNames }} {{ $dnsName }}{{- end }};
{{- end }}
ssl_certificate certs/{{ with $nginxServer.SecretItem.CertificatePath }}{{ . }}{{ else }}{{ $nginxServer.SecretItem.CertificateField }}{{ end }};
ssl_certificate_key certs/{{ with $nginxServer.SecretItem.KeyPath }}{{ . }}{{ else }}{{ $nginxServer.SecretItem.KeyField }}{{ end }};
{{- end }}
{{- if boolValue $config.CacheEnabled }}
Expand Down Expand Up @@ -469,5 +506,6 @@ http {
{{ template "server" .}}
}
{{- end}}
}
`
107 changes: 107 additions & 0 deletions internal/pkg/rpaas/nginx/configuration_render_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package nginx

import (
"crypto/x509"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -202,13 +203,80 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) {
},
},
},
FullCertificates: []CertificateData{
{
Certificate: &x509.Certificate{
DNSNames: []string{"example.org"},
},
SecretItem: nginxv1alpha1.TLSSecretItem{
CertificateField: "default.crt",
KeyField: "default.key",
},
},
},
},
assertion: func(t *testing.T, result string) {
assert.Regexp(t, `listen 8443 default_server ssl http2;`, result)
assert.Regexp(t, `ssl_certificate\s+certs/default.crt;`, result)
assert.Regexp(t, `ssl_certificate_key certs/default.key;`, result)
},
},
{
name: "with many certs actived",
data: ConfigurationData{
Config: &v1alpha1.NginxConfig{},
Instance: &v1alpha1.RpaasInstance{
Spec: v1alpha1.RpaasInstanceSpec{
Certificates: &nginxv1alpha1.TLSSecret{
SecretName: "secret-name",
Items: []nginxv1alpha1.TLSSecretItem{
{
CertificateField: "example.crt",
KeyField: "example.key",
},
{
CertificateField: "cert-manager.crt",
KeyField: "cert-manager.key",
},
},
},
},
},
FullCertificates: []CertificateData{
{
Certificate: &x509.Certificate{
DNSNames: []string{"example.org", "example.io"},
},
SecretItem: nginxv1alpha1.TLSSecretItem{
CertificateField: "example.crt",
KeyField: "example.key",
},
},
{
Certificate: &x509.Certificate{
DNSNames: []string{"example.namespace.system.internal.company.com"},
},
SecretItem: nginxv1alpha1.TLSSecretItem{
CertificateField: "cert-manager.crt",
KeyField: "cert-manager.key",
},
},
},
},
assertion: func(t *testing.T, result string) {
assert.Regexp(t, `listen 8443 default_server ssl http2;`, result)
assert.Regexp(t, `listen 8443 ssl http2;`, result)

assert.Regexp(t, `ssl_certificate\s+certs/cert-manager.crt;`, result)
assert.Regexp(t, `ssl_certificate_key certs/cert-manager.key;`, result)

assert.Regexp(t, `ssl_certificate\s+certs/example.crt;`, result)
assert.Regexp(t, `ssl_certificate_key certs/example.key;`, result)

assert.Regexp(t, `server_name example.org example.io;`, result)
assert.Regexp(t, `server_name example.namespace.system.internal.company.com;`, result)
},
},
{
name: "with TLS actived and custom listen options",
data: ConfigurationData{
Expand All @@ -230,6 +298,19 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) {
},
},
},
FullCertificates: []CertificateData{
{
Certificate: &x509.Certificate{
DNSNames: []string{"example.org"},
},
SecretItem: nginxv1alpha1.TLSSecretItem{
CertificateField: "default.crt",
CertificatePath: "custom_certificate_name.crt",
KeyField: "default.key",
KeyPath: "custom_key_name.key",
},
},
},
},
assertion: func(t *testing.T, result string) {
assert.Regexp(t, `listen 8443 default_server ssl http2 backlog=2048 deferred reuseport;`, result)
Expand Down Expand Up @@ -404,6 +485,19 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) {
},
},
},
FullCertificates: []CertificateData{
{
Certificate: &x509.Certificate{
DNSNames: []string{"example.org"},
},
SecretItem: nginxv1alpha1.TLSSecretItem{
CertificateField: "default.crt",
CertificatePath: "custom_certificate_name.crt",
KeyField: "default.key",
KeyPath: "custom_key_name.key",
},
},
},
},
assertion: func(t *testing.T, result string) {
assert.Regexp(t, `listen 80 default_server;`, result)
Expand Down Expand Up @@ -446,6 +540,19 @@ func TestRpaasConfigurationRenderer_Render(t *testing.T) {
},
},
},
FullCertificates: []CertificateData{
{
Certificate: &x509.Certificate{
DNSNames: []string{"example.org"},
},
SecretItem: nginxv1alpha1.TLSSecretItem{
CertificateField: "default.crt",
CertificatePath: "custom_certificate_name.crt",
KeyField: "default.key",
KeyPath: "custom_key_name.key",
},
},
},
},
assertion: func(t *testing.T, result string) {
assert.Regexp(t, `listen 20001 default_server;`, result)
Expand Down

0 comments on commit 8c46831

Please sign in to comment.