Skip to content

Commit

Permalink
feat: use KIC as a library in ControlPlane controller
Browse files Browse the repository at this point in the history
  • Loading branch information
@czeslavo
czeslavo committed Feb 20, 2025
1 parent ddaa6b8 commit 0d84b2c
Show file tree
Hide file tree
Showing 15 changed files with 408 additions and 1,032 deletions.
19 changes: 6 additions & 13 deletions config/rbac/role/role.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ rules:
- ""
resources:
- configmaps
- serviceaccounts
- services
verbs:
- create
Expand Down Expand Up @@ -50,6 +49,12 @@ rules:
- get
- list
- watch
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- delete
- apiGroups:
- ""
resources:
Expand All @@ -63,13 +68,7 @@ rules:
resources:
- validatingwebhookconfigurations
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apiextensions.k8s.io
resources:
Expand Down Expand Up @@ -449,10 +448,4 @@ rules:
- clusterrolebindings
- clusterroles
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
433 changes: 146 additions & 287 deletions controller/controlplane/controller.go
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions controller/controlplane/controller_consts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,7 @@ const (
ControlPlaneFinalizerCleanupClusterRoleBinding ControlPlaneFinalizer = "gateway-operator.konghq.com/cleanup-clusterrolebinding"
// ControlPlaneFinalizerCleanupValidatingWebhookConfiguration is the finalizer to cleanup validatingwebhookconfigurations owned by controlplane on deleting.
ControlPlaneFinalizerCleanupValidatingWebhookConfiguration ControlPlaneFinalizer = "gateway-operator.konghq.com/cleanup-validatingwebhookconfiguration"

// ControlPlaneFinalizerCPInstanceTeardown is the finalizer to tear down a controlplane instance.
ControlPlaneFinalizerCPInstanceTeardown ControlPlaneFinalizer = "gateway-operator.konghq.com/teardown-cp-instance"
)
7 changes: 0 additions & 7 deletions controller/controlplane/controller_rbac.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,3 @@ package controlplane
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=controlplanes,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=controlplanes/status,verbs=update;patch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=controlplanes/finalizers,verbs=update
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles,verbs=create;get;list;watch;update;patch;delete
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=create;get;list;watch;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;get;list;watch;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services,verbs=create;get;list;watch;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=create;get;list;watch;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete
242 changes: 0 additions & 242 deletions controller/controlplane/controller_reconciler_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/go-logr/logr"
"github.com/google/go-cmp/cmp"
"github.com/samber/lo"
admregv1 "k8s.io/api/admissionregistration/v1"
appsv1 "k8s.io/api/apps/v1"
certificatesv1 "k8s.io/api/certificates/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -441,58 +440,6 @@ func (r *Reconciler) ensureAdminMTLSCertificateSecret(
)
}

// ensureAdmissionWebhookCertificateSecret ensures that a Secret is created with the serving certificate for the
// ControlPlane's admission webhook.
func (r *Reconciler) ensureAdmissionWebhookCertificateSecret(
ctx context.Context,
logger logr.Logger,
cp *operatorv1beta1.ControlPlane,
admissionWebhookService *corev1.Service,
) (
op.Result,
*corev1.Secret,
error,
) {
usages := []certificatesv1.KeyUsage{
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageServerAuth,
certificatesv1.UsageDigitalSignature,
}
matchingLabels := client.MatchingLabels{
consts.SecretUsedByServiceLabel: consts.ControlPlaneServiceKindWebhook,
}
if !isAdmissionWebhookEnabled(ctx, r.Client, logger, cp) {
labels := k8sresources.GetManagedLabelForOwner(cp)
labels[consts.SecretUsedByServiceLabel] = consts.ControlPlaneServiceKindWebhook
secrets, err := k8sutils.ListSecretsForOwner(ctx, r.Client, cp.GetUID(), matchingLabels)
if err != nil {
return op.Noop, nil, fmt.Errorf("failed listing Secrets for ControlPlane %s/: %w", client.ObjectKeyFromObject(cp), err)
}
for _, svc := range secrets {
if err := r.Client.Delete(ctx, &svc); err != nil {
return op.Noop, nil, fmt.Errorf("failed deleting ControlPlane admission webhook Secret %s: %w", svc.Name, err)
}
}
if len(secrets) == 0 {
return op.Noop, nil, nil
}
return op.Deleted, nil, nil
}

return secrets.EnsureCertificate(ctx,
cp,
fmt.Sprintf("%s.%s.svc", admissionWebhookService.Name, admissionWebhookService.Namespace),
k8stypes.NamespacedName{
Namespace: r.ClusterCASecretNamespace,
Name: r.ClusterCASecretName,
},
usages,
r.ClusterCAKeyConfig,
r.Client,
matchingLabels,
)
}

// ensureOwnedClusterRolesDeleted removes all the owned ClusterRoles of the controlplane.
// it is called on cleanup of owned cluster resources on controlplane deletion.
// returns nil if all of owned ClusterRoles successfully deleted (ok if no owned CRs or NotFound on deleting CRs).
Expand Down Expand Up @@ -554,192 +501,3 @@ func (r *Reconciler) ensureOwnedClusterRoleBindingsDeleted(

return deleted, errors.Join(errs...)
}

func (r *Reconciler) ensureOwnedValidatingWebhookConfigurationDeleted(ctx context.Context,
cp *operatorv1beta1.ControlPlane,
) (deletions bool, err error) {
validatingWebhookConfigurations, err := k8sutils.ListValidatingWebhookConfigurations(
ctx,
r.Client,
client.MatchingLabels(k8sutils.GetManagedByLabelSet(cp)),
)
if err != nil {
return false, fmt.Errorf("failed listing webhook configurations for owner: %w", err)
}

var (
deleted bool
errs []error
)
for i := range validatingWebhookConfigurations {
if err = r.Client.Delete(ctx, &validatingWebhookConfigurations[i]); client.IgnoreNotFound(err) != nil {
errs = append(errs, err)
continue
}
deleted = true
}
return deleted, errors.Join(errs...)
}

func (r *Reconciler) ensureAdmissionWebhookService(
ctx context.Context,
logger logr.Logger,
cl client.Client,
cp *operatorv1beta1.ControlPlane,
) (op.Result, *corev1.Service, error) {
matchingLabels := k8sresources.GetManagedLabelForOwner(cp)
matchingLabels[consts.ControlPlaneServiceLabel] = consts.ControlPlaneServiceKindWebhook

services, err := k8sutils.ListServicesForOwner(
ctx,
cl,
cp.Namespace,
cp.UID,
matchingLabels,
)
if err != nil {
return op.Noop, nil, fmt.Errorf("failed listing admission webhook Services for ControlPlane %s/%s: %w", cp.Namespace, cp.Name, err)
}

if !isAdmissionWebhookEnabled(ctx, cl, logger, cp) {
for _, svc := range services {
if err := cl.Delete(ctx, &svc); client.IgnoreNotFound(err) != nil {
return op.Noop, nil, fmt.Errorf("failed deleting ControlPlane admission webhook Service %s: %w", svc.Name, err)
}
}
if len(services) == 0 {
return op.Noop, nil, nil
}
return op.Deleted, nil, nil
}

count := len(services)
if count > 1 {
if err := k8sreduce.ReduceServices(ctx, cl, services); err != nil {
return op.Noop, nil, err
}
return op.Noop, nil, errors.New("number of ControlPlane admission webhook Services reduced")
}

generatedService, err := k8sresources.GenerateNewAdmissionWebhookServiceForControlPlane(cp)
if err != nil {
return op.Noop, nil, err
}

if count == 1 {
var updated bool
existingService := &services[0]
updated, existingService.ObjectMeta = k8sutils.EnsureObjectMetaIsUpdated(existingService.ObjectMeta, generatedService.ObjectMeta)

if !cmp.Equal(existingService.Spec.Selector, generatedService.Spec.Selector) {
existingService.Spec.Selector = generatedService.Spec.Selector
updated = true
}
if !cmp.Equal(existingService.Spec.Ports, generatedService.Spec.Ports) {
existingService.Spec.Ports = generatedService.Spec.Ports
updated = true
}

if updated {
if err := cl.Update(ctx, existingService); err != nil {
return op.Noop, existingService, fmt.Errorf("failed updating ControlPlane admission webhook Service %s: %w", existingService.Name, err)
}
return op.Updated, existingService, nil
}
return op.Noop, existingService, nil
}

if err := cl.Create(ctx, generatedService); err != nil {
return op.Noop, nil, fmt.Errorf("failed creating ControlPlane admission webhook Service: %w", err)
}

return op.Created, generatedService, nil
}

func (r *Reconciler) ensureValidatingWebhookConfiguration(
ctx context.Context,
cp *operatorv1beta1.ControlPlane,
certSecret *corev1.Secret,
webhookService *corev1.Service,
) (op.Result, error) {
logger := log.GetLogger(ctx, "controlplane.ensureValidatingWebhookConfiguration", r.DevelopmentMode)

validatingWebhookConfigurations, err := k8sutils.ListValidatingWebhookConfigurations(
ctx,
r.Client,
client.MatchingLabels(k8sutils.GetManagedByLabelSet(cp)),
)
if err != nil {
return op.Noop, fmt.Errorf("failed listing webhook configurations for owner: %w", err)
}

count := len(validatingWebhookConfigurations)
if count > 1 {
if err := k8sreduce.ReduceValidatingWebhookConfigurations(ctx, r.Client, validatingWebhookConfigurations); err != nil {
return op.Noop, err
}
return op.Noop, errors.New("number of validatingWebhookConfigurations reduced")
}

if !isAdmissionWebhookEnabled(ctx, r.Client, logger, cp) {
for _, webhookConfiguration := range validatingWebhookConfigurations {
if err := r.Client.Delete(ctx, &webhookConfiguration); client.IgnoreNotFound(err) != nil {
return op.Noop, fmt.Errorf("failed deleting ControlPlane admission webhook ValidatingWebhookConfiguration %s: %w", webhookConfiguration.Name, err)
}
}
if len(validatingWebhookConfigurations) == 0 {
return op.Noop, nil
}
return op.Deleted, nil
}

cpContainer := k8sutils.GetPodContainerByName(&cp.Spec.Deployment.PodTemplateSpec.Spec, consts.ControlPlaneControllerContainerName)
if cpContainer == nil {
return op.Noop, errors.New("controller container not found")
}

caBundle, ok := certSecret.Data["ca.crt"]
if !ok {
return op.Noop, errors.New("ca.crt not found in secret")
}
generatedWebhookConfiguration, err := k8sresources.GenerateValidatingWebhookConfigurationForControlPlane(
cp.Name,
cpContainer.Image,
r.DevelopmentMode,
admregv1.WebhookClientConfig{
Service: &admregv1.ServiceReference{
Namespace: cp.Namespace,
Name: webhookService.GetName(),
Port: lo.ToPtr(int32(consts.ControlPlaneAdmissionWebhookListenPort)),
},
CABundle: caBundle,
},
)
if err != nil {
return op.Noop, fmt.Errorf("failed generating ControlPlane's ValidatingWebhookConfiguration: %w", err)
}
k8sutils.SetOwnerForObjectThroughLabels(generatedWebhookConfiguration, cp)

if count == 1 {
var updated bool
webhookConfiguration := validatingWebhookConfigurations[0]
old := webhookConfiguration.DeepCopy()

updated, webhookConfiguration.ObjectMeta = k8sutils.EnsureObjectMetaIsUpdated(webhookConfiguration.ObjectMeta, generatedWebhookConfiguration.ObjectMeta)

if !cmp.Equal(webhookConfiguration.Webhooks, generatedWebhookConfiguration.Webhooks) ||
!cmp.Equal(webhookConfiguration.Labels, generatedWebhookConfiguration.Labels) {
webhookConfiguration.Webhooks = generatedWebhookConfiguration.Webhooks
updated = true
}

if updated {
log.Debug(logger, "patching existing ValidatingWebhookConfiguration")
return op.Updated, r.Client.Patch(ctx, &webhookConfiguration, client.MergeFrom(old))
}

return op.Noop, nil
}

return op.Created, r.Client.Create(ctx, generatedWebhookConfiguration)
}
Loading

0 comments on commit 0d84b2c

Please sign in to comment.