From 2123d975fa49b16675cf38752076916d903fac91 Mon Sep 17 00:00:00 2001 From: "Alessio Giliberti, Chiara Mercurio" Date: Tue, 3 Dec 2024 17:09:46 +0100 Subject: [PATCH 1/5] Add PVC Provisioning Job --- .../tenant-controller/tenant_controller.go | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/operators/pkg/tenant-controller/tenant_controller.go b/operators/pkg/tenant-controller/tenant_controller.go index 5b8908456..09907a32b 100644 --- a/operators/pkg/tenant-controller/tenant_controller.go +++ b/operators/pkg/tenant-controller/tenant_controller.go @@ -22,6 +22,7 @@ import ( "strings" "time" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -30,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -53,6 +55,10 @@ const ( NFSSecretServerNameKey = "server-name" // NFSSecretPathKey -> NFS path key in NFS secret. NFSSecretPathKey = "path" + // ProvisionJobBaseImage -> Base container image for Personal Drive provision job + ProvisionJobBaseImage = "busybox" + // ProvisionJobLabel -> Key of the label added by the Provision Job + ProvisionJobLabel = "pvc-provisioning" ) // TenantReconciler reconciles a Tenant object. @@ -243,6 +249,7 @@ func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rbacv1.ClusterRole{}). Owns(&rbacv1.ClusterRoleBinding{}). Owns(&netv1.NetworkPolicy{}). + Owns(&batchv1.Job{}). Watches(&crownlabsv1alpha1.Workspace{}, handler.EnqueueRequestsFromMapFunc(r.workspaceToEnrolledTenants)). WithOptions(controller.Options{ @@ -517,6 +524,67 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context return err } klog.Infof("PVC Secret for tenant %s %s", tn.Name, pvcSecOpRes) + + val, ok := pvc.Labels[ProvisionJobLabel] + if !ok || val != "completed" { + chownJob := batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: myDrivePVCName(tn.Name) + "-provision", Namespace: r.MyDrivePVCsNamespace}} + + chownJobOpRes, err := ctrl.CreateOrUpdate(ctx, r.Client, &chownJob, func() error { + + if chownJob.CreationTimestamp.IsZero() { + chownJob.Spec.BackoffLimit = ptr.To[int32](3) + chownJob.Spec.Template.Spec.RestartPolicy = v1.RestartPolicyNever + chownJob.Spec.Template.Spec.Containers = []v1.Container{ + { + Name: "chown-container", + Image: ProvisionJobBaseImage, + Command: []string{"chown", "-R", fmt.Sprintf("%d", forge.CrownLabsUserID) + ":" + fmt.Sprintf("%d", forge.CrownLabsUserID), "/mnt/mydrive"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "mydrive", + MountPath: "/mnt/mydrive", + }, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("128Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("256Mi"), + }, + }, + }, + } + chownJob.Spec.Template.Spec.Volumes = []v1.Volume{ + { + Name: "mydrive", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }, + } + } + + if chownJob.Status.Succeeded == 1 { + pvc.Labels[ProvisionJobLabel] = "completed" + klog.Infof("PVC Provisioning Job completed for tenant %s", tn.Name) + } else if chownJob.Status.Failed == 1 { + klog.Errorf("PVC Provisioning Job failed for tenant %s", tn.Name) + } + + return ctrl.SetControllerReference(tn, &chownJob, r.Scheme) + }) + if err != nil { + klog.Errorf("Unable to create or update PVC Provisioning Job for tenant %s -> %s", tn.Name, err) + return err + } + klog.Infof("PVC Provisioning Job for tenant %s %s", tn.Name, chownJobOpRes) + } + } else if pvc.Status.Phase == v1.ClaimPending { klog.Infof("PVC pending for tenant %s", tn.Name) } From 74e3ad8abebad09ddba061123d02ec81951bebad Mon Sep 17 00:00:00 2001 From: "Alessio Giliberti, Chiara Mercurio" Date: Fri, 13 Dec 2024 10:40:42 +0100 Subject: [PATCH 2/5] Added constants, moved ProvisionJob spec in separate function, minor fixes --- .../tenant-controller/tenant_controller.go | 107 ++++++++++-------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/operators/pkg/tenant-controller/tenant_controller.go b/operators/pkg/tenant-controller/tenant_controller.go index 09907a32b..8ac8ccea6 100644 --- a/operators/pkg/tenant-controller/tenant_controller.go +++ b/operators/pkg/tenant-controller/tenant_controller.go @@ -57,8 +57,14 @@ const ( NFSSecretPathKey = "path" // ProvisionJobBaseImage -> Base container image for Personal Drive provision job ProvisionJobBaseImage = "busybox" - // ProvisionJobLabel -> Key of the label added by the Provision Job + // ProvisionJobLabel -> Key of the label added by the Provision Job to flag the PVC ProvisionJobLabel = "pvc-provisioning" + // ProvisionJobValue -> Value of the label added by the Provision Job to flag the PVC + ProvisionJobValue = "completed" + // ProvisionJobMaxRetries -> Maximum number of retries for Provision jobs + ProvisionJobMaxRetries = 3 + // ProvisionJobTTLSeconds -> Seconds for Provision jobs before deletion (either failure or success) + ProvisionJobTTLSeconds = 604800 ) // TenantReconciler reconciles a Tenant object. @@ -526,56 +532,11 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context klog.Infof("PVC Secret for tenant %s %s", tn.Name, pvcSecOpRes) val, ok := pvc.Labels[ProvisionJobLabel] - if !ok || val != "completed" { - chownJob := batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: myDrivePVCName(tn.Name) + "-provision", Namespace: r.MyDrivePVCsNamespace}} + if !ok || val != ProvisionJobValue { + chownJob := batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: pvc.Name + "-provision", Namespace: pvc.Namespace}} chownJobOpRes, err := ctrl.CreateOrUpdate(ctx, r.Client, &chownJob, func() error { - - if chownJob.CreationTimestamp.IsZero() { - chownJob.Spec.BackoffLimit = ptr.To[int32](3) - chownJob.Spec.Template.Spec.RestartPolicy = v1.RestartPolicyNever - chownJob.Spec.Template.Spec.Containers = []v1.Container{ - { - Name: "chown-container", - Image: ProvisionJobBaseImage, - Command: []string{"chown", "-R", fmt.Sprintf("%d", forge.CrownLabsUserID) + ":" + fmt.Sprintf("%d", forge.CrownLabsUserID), "/mnt/mydrive"}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "mydrive", - MountPath: "/mnt/mydrive", - }, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("128Mi"), - }, - Limits: v1.ResourceList{ - "cpu": resource.MustParse("500m"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - } - chownJob.Spec.Template.Spec.Volumes = []v1.Volume{ - { - Name: "mydrive", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - }, - }, - }, - } - } - - if chownJob.Status.Succeeded == 1 { - pvc.Labels[ProvisionJobLabel] = "completed" - klog.Infof("PVC Provisioning Job completed for tenant %s", tn.Name) - } else if chownJob.Status.Failed == 1 { - klog.Errorf("PVC Provisioning Job failed for tenant %s", tn.Name) - } - + r.updateTnProvisioningJob(&chownJob, &pvc) return ctrl.SetControllerReference(tn, &chownJob, r.Scheme) }) if err != nil { @@ -583,6 +544,16 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context return err } klog.Infof("PVC Provisioning Job for tenant %s %s", tn.Name, chownJobOpRes) + + if chownJob.Status.Succeeded == 1 { + ctrl.CreateOrUpdate(ctx, r.Client, &pvc, func() error { + pvc.Labels[ProvisionJobLabel] = ProvisionJobValue + return nil + }) + klog.Infof("PVC Provisioning Job completed for tenant %s", tn.Name) + } else if chownJob.Status.Failed == 1 { + klog.Errorf("PVC Provisioning Job failed for tenant %s", tn.Name) + } } } else if pvc.Status.Phase == v1.ClaimPending { @@ -644,6 +615,44 @@ func (r *TenantReconciler) updateTnPersistentVolumeClaim(pvc *v1.PersistentVolum pvc.Spec.StorageClassName = &scName } +func (r *TenantReconciler) updateTnProvisioningJob(chownJob *batchv1.Job, pvc *v1.PersistentVolumeClaim) { + if chownJob.CreationTimestamp.IsZero() { + chownJob.Spec.BackoffLimit = ptr.To[int32](ProvisionJobMaxRetries) + chownJob.Spec.TTLSecondsAfterFinished = ptr.To[int32](ProvisionJobTTLSeconds) + chownJob.Spec.Template.Spec.RestartPolicy = v1.RestartPolicyOnFailure + chownJob.Spec.Template.Spec.Containers = []v1.Container{{ + Name: "chown-container", + Image: ProvisionJobBaseImage, + Command: []string{"chown", "-R", fmt.Sprintf("%d:%d", forge.CrownLabsUserID, forge.CrownLabsUserID), forge.MyDriveVolumeMountPath}, + VolumeMounts: []v1.VolumeMount{{ + Name: "mydrive", + MountPath: forge.MyDriveVolumeMountPath, + }, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("128Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("128Mi"), + }, + }, + }, + } + chownJob.Spec.Template.Spec.Volumes = []v1.Volume{{ + Name: "mydrive", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }, + } + } +} + func (r *TenantReconciler) handleKeycloakSubscription(ctx context.Context, tn *crownlabsv1alpha2.Tenant, tenantExistingWorkspaces []crownlabsv1alpha2.TenantWorkspaceEntry) error { // KcA could be nil for local testing skipping the keycloak subscription if r.KcA == nil { From 72d8054fc7f090699581cab469e229b8cc0ce1d5 Mon Sep 17 00:00:00 2001 From: "Alessio Giliberti, Chiara Mercurio" Date: Wed, 18 Dec 2024 10:56:00 +0100 Subject: [PATCH 3/5] Added Pending label, Linting issues --- .../tenant-controller/tenant_controller.go | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/operators/pkg/tenant-controller/tenant_controller.go b/operators/pkg/tenant-controller/tenant_controller.go index 8ac8ccea6..d4b912c0e 100644 --- a/operators/pkg/tenant-controller/tenant_controller.go +++ b/operators/pkg/tenant-controller/tenant_controller.go @@ -55,15 +55,17 @@ const ( NFSSecretServerNameKey = "server-name" // NFSSecretPathKey -> NFS path key in NFS secret. NFSSecretPathKey = "path" - // ProvisionJobBaseImage -> Base container image for Personal Drive provision job + // ProvisionJobBaseImage -> Base container image for Personal Drive provision job. ProvisionJobBaseImage = "busybox" - // ProvisionJobLabel -> Key of the label added by the Provision Job to flag the PVC - ProvisionJobLabel = "pvc-provisioning" - // ProvisionJobValue -> Value of the label added by the Provision Job to flag the PVC - ProvisionJobValue = "completed" - // ProvisionJobMaxRetries -> Maximum number of retries for Provision jobs + // ProvisionJobLabel -> Key of the label added by the Provision Job to flag the PVC. + ProvisionJobLabel = "mydrive-provisioning" + // ProvisionJobValueOk -> Value of the label added by the Provision Job to flag the PVC when everything worked fine. + ProvisionJobValueOk = "completed" + // ProvisionJobValuePending -> Value of the label added by the Provision Job to flag the PVC when it hasn't completed yet. + ProvisionJobValuePending = "pending" + // ProvisionJobMaxRetries -> Maximum number of retries for Provision jobs. ProvisionJobMaxRetries = 3 - // ProvisionJobTTLSeconds -> Seconds for Provision jobs before deletion (either failure or success) + // ProvisionJobTTLSeconds -> Seconds for Provision jobs before deletion (either failure or success). ProvisionJobTTLSeconds = 604800 ) @@ -532,7 +534,7 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context klog.Infof("PVC Secret for tenant %s %s", tn.Name, pvcSecOpRes) val, ok := pvc.Labels[ProvisionJobLabel] - if !ok || val != ProvisionJobValue { + if !ok || val != ProvisionJobValueOk { chownJob := batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: pvc.Name + "-provision", Namespace: pvc.Namespace}} chownJobOpRes, err := ctrl.CreateOrUpdate(ctx, r.Client, &chownJob, func() error { @@ -546,16 +548,16 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context klog.Infof("PVC Provisioning Job for tenant %s %s", tn.Name, chownJobOpRes) if chownJob.Status.Succeeded == 1 { - ctrl.CreateOrUpdate(ctx, r.Client, &pvc, func() error { - pvc.Labels[ProvisionJobLabel] = ProvisionJobValue - return nil - }) + pvc.Labels[ProvisionJobLabel] = ProvisionJobValueOk + if err := r.Update(ctx, &pvc); err != nil { + klog.Errorf("PVC Provisioning Job failed to update PVC labels for tenant %s", tn.Name) + } + klog.Infof("PVC Provisioning Job completed for tenant %s", tn.Name) } else if chownJob.Status.Failed == 1 { klog.Errorf("PVC Provisioning Job failed for tenant %s", tn.Name) } } - } else if pvc.Status.Phase == v1.ClaimPending { klog.Infof("PVC pending for tenant %s", tn.Name) } @@ -609,6 +611,7 @@ func (r *TenantReconciler) updateTnNetPolAllow(np *netv1.NetworkPolicy) { func (r *TenantReconciler) updateTnPersistentVolumeClaim(pvc *v1.PersistentVolumeClaim) { scName := r.MyDrivePVCsStorageClassName pvc.Labels = r.updateTnResourceCommonLabels(pvc.Labels) + pvc.Labels[ProvisionJobLabel] = ProvisionJobValuePending pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} pvc.Spec.Resources.Requests = v1.ResourceList{v1.ResourceStorage: r.MyDrivePVCsSize} From 4f84267db09a12e167cefee87e66c0aa27e6703f Mon Sep 17 00:00:00 2001 From: "Alessio Giliberti, Chiara Mercurio" Date: Tue, 14 Jan 2025 23:06:22 +0100 Subject: [PATCH 4/5] Moved MyDrive provisioning labels, changed label logic --- operators/pkg/forge/labels.go | 7 ++++++ .../tenant-controller/tenant_controller.go | 22 +++++++++---------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/operators/pkg/forge/labels.go b/operators/pkg/forge/labels.go index e92518c98..2aab5b492 100644 --- a/operators/pkg/forge/labels.go +++ b/operators/pkg/forge/labels.go @@ -37,10 +37,17 @@ const ( InstanceSubmissionSelectorLabel = "crownlabs.polito.it/instance-submission-requested" // InstanceSubmissionCompletedLabel -> label for Instances that have been submitted. InstanceSubmissionCompletedLabel = "crownlabs.polito.it/instance-submission-completed" + // ProvisionJobLabel -> Key of the label added by the Provision Job to flag the Tenant's MyDrive PVC. + ProvisionJobLabel = "crownlabs.polito.it/mydrive-provisioning" labelManagedByInstanceValue = "instance" labelManagedByTenantValue = "tenant" labelTypeSandboxValue = "sandbox" + + // ProvisionJobValueOk -> Value of the label added by the Provision Job to flag the PVC when everything worked fine. + ProvisionJobValueOk = "completed" + // ProvisionJobValuePending -> Value of the label added by the Provision Job to flag the PVC when it hasn't completed yet. + ProvisionJobValuePending = "pending" ) // InstanceLabels receives in input a set of labels and returns the updated set depending on the specified template, diff --git a/operators/pkg/tenant-controller/tenant_controller.go b/operators/pkg/tenant-controller/tenant_controller.go index d4b912c0e..dbd9e6d56 100644 --- a/operators/pkg/tenant-controller/tenant_controller.go +++ b/operators/pkg/tenant-controller/tenant_controller.go @@ -57,12 +57,6 @@ const ( NFSSecretPathKey = "path" // ProvisionJobBaseImage -> Base container image for Personal Drive provision job. ProvisionJobBaseImage = "busybox" - // ProvisionJobLabel -> Key of the label added by the Provision Job to flag the PVC. - ProvisionJobLabel = "mydrive-provisioning" - // ProvisionJobValueOk -> Value of the label added by the Provision Job to flag the PVC when everything worked fine. - ProvisionJobValueOk = "completed" - // ProvisionJobValuePending -> Value of the label added by the Provision Job to flag the PVC when it hasn't completed yet. - ProvisionJobValuePending = "pending" // ProvisionJobMaxRetries -> Maximum number of retries for Provision jobs. ProvisionJobMaxRetries = 3 // ProvisionJobTTLSeconds -> Seconds for Provision jobs before deletion (either failure or success). @@ -533,10 +527,10 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context } klog.Infof("PVC Secret for tenant %s %s", tn.Name, pvcSecOpRes) - val, ok := pvc.Labels[ProvisionJobLabel] - if !ok || val != ProvisionJobValueOk { - chownJob := batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: pvc.Name + "-provision", Namespace: pvc.Namespace}} + chownJob := batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: pvc.Name + "-provision", Namespace: pvc.Namespace}} + val, found := pvc.Labels[forge.ProvisionJobLabel] + if !found { chownJobOpRes, err := ctrl.CreateOrUpdate(ctx, r.Client, &chownJob, func() error { r.updateTnProvisioningJob(&chownJob, &pvc) return ctrl.SetControllerReference(tn, &chownJob, r.Scheme) @@ -545,17 +539,22 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context klog.Errorf("Unable to create or update PVC Provisioning Job for tenant %s -> %s", tn.Name, err) return err } + pvc.Labels[forge.ProvisionJobLabel] = forge.ProvisionJobValuePending klog.Infof("PVC Provisioning Job for tenant %s %s", tn.Name, chownJobOpRes) + } else if val != forge.ProvisionJobValueOk { + if err := r.Update(ctx, &chownJob); err != nil { + klog.Errorf("PVC Provisioning Job failed to update info about Provisioning Job for tenant %s", tn.Name) + } if chownJob.Status.Succeeded == 1 { - pvc.Labels[ProvisionJobLabel] = ProvisionJobValueOk + pvc.Labels[forge.ProvisionJobLabel] = forge.ProvisionJobValueOk if err := r.Update(ctx, &pvc); err != nil { klog.Errorf("PVC Provisioning Job failed to update PVC labels for tenant %s", tn.Name) } klog.Infof("PVC Provisioning Job completed for tenant %s", tn.Name) } else if chownJob.Status.Failed == 1 { - klog.Errorf("PVC Provisioning Job failed for tenant %s", tn.Name) + klog.Warningf("PVC Provisioning Job failed for tenant %s", tn.Name) } } } else if pvc.Status.Phase == v1.ClaimPending { @@ -611,7 +610,6 @@ func (r *TenantReconciler) updateTnNetPolAllow(np *netv1.NetworkPolicy) { func (r *TenantReconciler) updateTnPersistentVolumeClaim(pvc *v1.PersistentVolumeClaim) { scName := r.MyDrivePVCsStorageClassName pvc.Labels = r.updateTnResourceCommonLabels(pvc.Labels) - pvc.Labels[ProvisionJobLabel] = ProvisionJobValuePending pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} pvc.Spec.Resources.Requests = v1.ResourceList{v1.ResourceStorage: r.MyDrivePVCsSize} From bfee4a0a699b7104f1e9e92bf319cde726f7f0da Mon Sep 17 00:00:00 2001 From: "Alessio Giliberti, Chiara Mercurio" Date: Wed, 15 Jan 2025 19:05:37 +0100 Subject: [PATCH 5/5] Forgot "pending" label update --- operators/pkg/tenant-controller/tenant_controller.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/operators/pkg/tenant-controller/tenant_controller.go b/operators/pkg/tenant-controller/tenant_controller.go index dbd9e6d56..8ad51bb72 100644 --- a/operators/pkg/tenant-controller/tenant_controller.go +++ b/operators/pkg/tenant-controller/tenant_controller.go @@ -540,6 +540,10 @@ func (r *TenantReconciler) createOrUpdateTnPersonalNFSVolume(ctx context.Context return err } pvc.Labels[forge.ProvisionJobLabel] = forge.ProvisionJobValuePending + if err := r.Update(ctx, &pvc); err != nil { + klog.Errorf("PVC Provisioning Job failed to update PVC labels for tenant %s", tn.Name) + } + klog.Infof("PVC Provisioning Job for tenant %s %s", tn.Name, chownJobOpRes) } else if val != forge.ProvisionJobValueOk { if err := r.Update(ctx, &chownJob); err != nil {