feat: Move supported resources from hard-coded to ConfigMap

Signed-off-by: maromcohen <marom@dana.org>

.gitignore

@@ -26,3 +26,5 @@ go.work
 *.swp
 *.swo
 *~
+
+*.uuid

config/configMap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sns-quota-resources
+  namespace: sns-system
+data:
+  resources: "basic.storageclass.storage.k8s.io/requests.storage,cpu,memory,pods,requests.nvidia.com/gpu"

config/manager/kustomization.yaml

@@ -15,3 +15,9 @@ configMapGenerator:
   name: config
   options:
     disableNameSuffixHash: true
+- literals:
+  - resources="basic.storageclass.storage.k8s.io/requests.storage,cpu,memory,pods,requests.nvidia.com/gpu"
+  name: quota-resources
+  namespace: sns-system
+  options:
+    disableNameSuffixHash: true

config/rbac/configmap_role.yaml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: configmap-role
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch"]

config/rbac/configmap_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: configmap-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: configmap-role
+subjects:
+  - kind: ServiceAccount
+    name: sns-controller-manager
+    namespace: sns-system

config/rbac/role.yaml

@@ -4,14 +4,6 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - ""
   resources:
@@ -48,6 +40,14 @@ rules:
   - users
   verbs:
   - impersonate
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

internal/quota/default.go

@@ -1,16 +1,18 @@
 package quota
 
 import (
+	"context"
+	"fmt"
+	"strings"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
-	BasicStorage corev1.ResourceName = "basic.storageclass.storage.k8s.io/requests.storage"
-	CPU          corev1.ResourceName = "cpu"
-	Memory       corev1.ResourceName = "memory"
-	Pods         corev1.ResourceName = "pods"
-	GPU          corev1.ResourceName = "requests.nvidia.com/gpu"
+	resourcesKey string = "resources"
 )
 
 var (
@@ -30,14 +32,29 @@ var (
 	imagestreams      = resource.NewQuantity(100, resource.DecimalSI)
 	ZeroDecimal       = resource.NewQuantity(0, resource.DecimalSI)
 
+	quotaConfig = "sns-quota-resources"
+
 	DefaultQuota = corev1.ResourceQuotaSpec{Hard: DefaultQuotaHard}
 
 	DefaultQuotaHard = corev1.ResourceList{"configmaps": *Configmaps, "count/builds.build.openshift.io": *Builds, "count/cronjobs.batch": *Cronjobs, "count/daemonsets.apps": *Daemonsets,
 		"count/deployments.apps": *Deployments, "count/jobs.batch": *Cronjobs, "count/replicasets.apps": *Replicasets, "count/routes.route.openshift.io": *Routes,
 		"secrets": *Secrets, "count/deploymentconfigs.apps.openshift.io": *deploymentconfigs, "count/buildconfigs.build.openshift.io": *buildconfigs, "count/serviceaccounts": *serviceaccounts,
 		"count/statefulsets.apps": *statefulsets, "count/templates.template.openshift.io": *templates, "openshift.io/imagestreams": *imagestreams}
+)
 
-	ZeroedQuota = corev1.ResourceQuotaSpec{
-		Hard: corev1.ResourceList{BasicStorage: *ZeroDecimal, CPU: *ZeroDecimal, Memory: *ZeroDecimal, Pods: *ZeroDecimal, GPU: *ZeroDecimal},
+// GetObservedResources returns default values for all observed resources inside a ResourceQuotaSpec object.
+// The observed resources are read from a configMap.
+func GetObservedResources(ctx context.Context, k8sClient client.Client) (corev1.ResourceQuotaSpec, error) {
+	resourcesConfig := &corev1.ConfigMap{}
+	if err := k8sClient.Get(ctx, types.NamespacedName{Name: quotaConfig, Namespace: namespaceName}, resourcesConfig); err != nil {
+		return corev1.ResourceQuotaSpec{}, fmt.Errorf("failed to get ConfigMap %q: %v", resourcesConfig, err)
 	}
-)
+
+	resources := corev1.ResourceList{}
+	resourceNames := strings.Split(resourcesConfig.Data[resourcesKey], ",")
+	for _, name := range resourceNames {
+		resources[corev1.ResourceName(name)] = *ZeroDecimal
+	}
+
+	return corev1.ResourceQuotaSpec{Hard: resources}, nil
+}

internal/quota/resources.go

@@ -28,10 +28,10 @@ func ResourceListEqual(resourceListA, resourceListB corev1.ResourceList) bool {
 }
 
 // ResourceQuotaSpecEqual gets two ResourceQuotaSpecs and returns whether their specs are equal.
-func ResourceQuotaSpecEqual(resourceQuotaSpecA, resourceQuotaSpecB corev1.ResourceQuotaSpec) bool {
+func ResourceQuotaSpecEqual(resourceQuotaSpecA, resourceQuotaSpecB corev1.ResourceQuotaSpec, observedResourcesSpec corev1.ResourceQuotaSpec) bool {
 	var resources []string
 
-	for resourceName := range ZeroedQuota.Hard {
+	for resourceName := range observedResourcesSpec.Hard {
 		resources = append(resources, resourceName.String())
 	}
 

internal/quota/snsquota.go

@@ -38,10 +38,15 @@ func EnsureSubnamespaceObject(snsObject *objectcontext.ObjectContext, isRq bool)
 	quotaObjectName := snsObject.Name()
 	quotaSpec := SubnamespaceSpec(snsObject.Object)
 
+	observedResources, err := GetObservedResources(snsObject.Ctx, snsObject.Client)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// if the subnamespace does not have a quota in its Spec, then set it to be equal to what it
 	// currently uses, and create a quotaObject for it. This can happen when converting a ResourcePool to an SNS
 	if len(quotaSpec.Hard) == 0 {
-		quotaObject, err := setupObject(quotaObjectName, isRq, ZeroedQuota, snsObject)
+		quotaObject, err := setupObject(quotaObjectName, isRq, observedResources, snsObject)
 		if err != nil {
 			return ctrl.Result{}, err
 		}

internal/subnamespace/commonvalidator.go

@@ -15,7 +15,13 @@ import (
 // In a ResourcePool it validates that the upper resource pool cannot be created with an empty quota.
 func ValidateResourceQuotaParams(snsObject *objectcontext.ObjectContext, isSNSResourcePool bool) admission.Response {
 	snsQuota := quota.SubnamespaceSpec(snsObject.Object).Hard
-	resourceQuotaParams := quota.ZeroedQuota.Hard
+
+	observedResources, err := quota.GetObservedResources(snsObject.Ctx, snsObject.Client)
+	if err != nil {
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	resourceQuotaParams := observedResources.Hard
 
 	if isSNSResourcePool {
 		if response := validateUpperResourcePool(snsObject, snsQuota); !response.Allowed {

internal/subnamespace/controller.go

@@ -36,7 +36,7 @@ type snsPhaseFunc func(*objectcontext.ObjectContext, *objectcontext.ObjectContex
 // +kubebuilder:rbac:groups="",resources=resourcequotas,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="quota.openshift.io",resources=clusterresourcequotas,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=limitranges,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;
+// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch
 
 // SetupWithManager sets up the controller by specifying the following: controller is managing the reconciliation
 // of subnamespace objects and is watching for changes to the SNSEvents channel and enqueues requests for the

internal/subnamespace/sync.go

@@ -1,6 +1,7 @@
 package subnamespace
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/dana-team/hns/internal/metrics"
@@ -133,7 +134,7 @@ func (r *SubnamespaceReconciler) sync(snsParentNS, snsObject *objectcontext.Obje
 	}
 	logger.Info("successfully ensured presence in namespacedb for subnamespace", "subnamespace", snsObject.Name())
 
-	if IsUpdateNeeded(snsObject.Object, childrenRequests, resourceAllocatedToChildren, free) {
+	if IsUpdateNeeded(snsObject.Ctx, snsObject.Client, snsObject.Object, childrenRequests, resourceAllocatedToChildren, free) {
 		if err := updateSNSResourcesStatus(snsObject, childrenRequests, resourceAllocatedToChildren, free); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to set status for subnamespace %q: %v", snsName, err.Error())
 		}
@@ -394,8 +395,8 @@ func (r *SubnamespaceReconciler) enqueueChildrenSNSToRPConversionEvents(snsObjec
 
 // IsUpdateNeeded gets a subnamespace object, a []danav1.Namespaces and two resource lists and returns whether
 // the subnamespace object status has to be updated.
-func IsUpdateNeeded(sns client.Object, childrenRequests []danav1.Namespaces, allocated, free corev1.ResourceList) bool {
-	if !NamespacesEqual(sns.(*danav1.Subnamespace).Status.Namespaces, childrenRequests) ||
+func IsUpdateNeeded(ctx context.Context, k8sClient client.Client, sns client.Object, childrenRequests []danav1.Namespaces, allocated, free corev1.ResourceList) bool {
+	if !NamespacesEqual(sns.(*danav1.Subnamespace).Status.Namespaces, childrenRequests, ctx, k8sClient) ||
 		!quota.ResourceListEqual(sns.(*danav1.Subnamespace).Status.Total.Allocated, allocated) ||
 		!quota.ResourceListEqual(sns.(*danav1.Subnamespace).Status.Total.Free, free) {
 		return true
@@ -404,12 +405,16 @@ func IsUpdateNeeded(sns client.Object, childrenRequests []danav1.Namespaces, all
 }
 
 // NamespacesEqual gets two []danav1.Namespaces and returns whether they are equal.
-func NamespacesEqual(nsA, nsB []danav1.Namespaces) bool {
+func NamespacesEqual(nsA, nsB []danav1.Namespaces, snsContext context.Context, snsClient client.Client) bool {
 	if len(nsA) != len(nsB) {
 		return false
 	}
+	observedResources, err := quota.GetObservedResources(snsContext, snsClient)
+	if err != nil {
+		return false
+	}
 	for i, nameQuotaPair := range nsA {
-		if !quota.ResourceQuotaSpecEqual(nameQuotaPair.ResourceQuotaSpec, nsB[i].ResourceQuotaSpec) {
+		if !quota.ResourceQuotaSpecEqual(nameQuotaPair.ResourceQuotaSpec, nsB[i].ResourceQuotaSpec, observedResources) {
 			return false
 		}
 	}