updating autoscaling

dabble-of-devops-bioanalyze · Sep 6, 2021 · bd3c4be · bd3c4be
1 parent 6b56160
commit bd3c4be
Show file tree

Hide file tree

Showing 5 changed files with 134 additions and 19 deletions.
diff --git a/helm_charts/autoscaler/values.yml.tpl b/helm_charts/autoscaler/values.yml.tpl
@@ -0,0 +1,20 @@
+
+#k8s_service_account_namespace = "kube-system"
+#k8s_service_account_name      = "cluster-autoscaler-aws-cluster-autoscaler"
+
+cloudProvider: aws
+awsRegion: ${region} 
+
+rbac:
+  create: true
+  serviceAccount:
+    # This value should match local.k8s_service_account_name in locals.tf
+    name: "${service_account_name}"
+    annotations:
+      # This value should match the ARN of the role created by module.iam_assumable_role_admin in irsa.tf
+      eks.amazonaws.com/role-arn: "${role_arn}"
+
+
+autoDiscovery:
+  clusterName: "${cluster_name}"
+  enabled: true
diff --git a/irsa.tf b/irsa.tf
@@ -1,15 +1,39 @@
 // TODO 
 // I am not sure if this is needed or not
-// I think the autoscaling is created through the cloudposse var.enable_autoscaling
+// I think the autoscaling is created through the cloudposse autoscaling policies 
 // https://github.com/cloudposse/terraform-aws-ec2-autoscale-group/blob/0.27.0/autoscaling.tf
+// But for now we'll keep the IRSA
+// Instructions are the same as in the main irsa example. We're just using a helm_resource to deploy the autoscaling charts 
+// https://github.com/terraform-aws-modules/terraform-aws-eks/tree/v17.11.0/examples/irsa
+
+locals {
+  eks_cluster_oidc_issuer_url = module.eks_cluster.eks_cluster_identity_oidc_issuer
+}
+
+module "iam_assumable_role_admin" {
+  source       = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version      = "3.6.0"
+  create_role  = true
+  role_name    = "cluster-autoscaler"
+  provider_url = replace(local.eks_cluster_oidc_issuer_url, "https://", "")
+
+  role_policy_arns = [
+  aws_iam_policy.cluster_autoscaler.arn]
+  oidc_fully_qualified_subjects = [
+  "system:serviceaccount:${local.k8s_service_account_namespace}:${local.k8s_service_account_name}"]
+}
 
 
 resource "aws_iam_policy" "cluster_autoscaler" {
   name_prefix = "cluster-autoscaler"
-  description = "EKS cluster-autoscaler policy for cluster ${module.label.id}"
+  description = "EKS cluster-autoscaler policy for cluster ${module.eks_cluster.eks_cluster_id}"
   policy      = data.aws_iam_policy_document.cluster_autoscaler.json
 }
 
+output "aws_iam_policy_cluster_autoscaler" {
+  value = aws_iam_policy.cluster_autoscaler.arn
+}
+
 data "aws_iam_policy_document" "cluster_autoscaler" {
   statement {
     sid    = "clusterAutoscalerAll"
@@ -40,8 +64,8 @@ data "aws_iam_policy_document" "cluster_autoscaler" {
 
     condition {
       test     = "StringEquals"
-      variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.label.id}"
-      values   = ["shared"]
+      variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.eks_cluster.eks_cluster_id}"
+      values   = ["owned"]
     }
 
     condition {
@@ -51,3 +75,40 @@ data "aws_iam_policy_document" "cluster_autoscaler" {
     }
   }
 }
+
+data "template_file" "autoscaler" {
+  depends_on = [
+    module.eks_cluster,
+    aws_iam_policy.cluster_autoscaler
+  ]
+  template = file("${path.module}/helm_charts/autoscaler/values.yml.tpl")
+  vars = {
+    region               = var.region
+    current_account      = data.aws_caller_identity.current.account_id
+    cluster_name         = module.eks_cluster.eks_cluster_id
+    role_arn             = module.iam_assumable_role_admin.this_iam_role_arn
+    service_account_name = local.k8s_service_account_name
+  }
+}
+
+# helm repo add autoscaler https://kubernetes.github.io/autoscaler
+#  helm install my-release autoscaler/cluster-autoscaler \
+# --set 'autoDiscovery.clusterName'=<CLUSTER NAME>
+
+resource "helm_release" "autoscaler" {
+  depends_on = [
+    module.eks_cluster,
+    data.template_file.autoscaler
+  ]
+  name       = "autoscaler"
+  repository = "https://kubernetes.github.io/autoscaler"
+  chart      = "cluster-autoscaler"
+  version    = "9.10.5"
+  namespace  = "kube-system"
+
+
+  values = [
+    data.template_file.autoscaler.rendered
+  ]
+}
+
diff --git a/main.tf b/main.tf
@@ -8,9 +8,12 @@ module "label" {
   version    = "0.24.1"
   attributes = ["cluster"]
 
+  # https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html
   tags = {
-    "k8s.io/cluster-autoscaler/${module.this.id}" = "true"
-    "k8s.io/cluster-autoscaler/enabled"           = "true"
+    "k8s.io/cluster-autoscaler/${module.this.id}"         = "owned"
+    "k8s.io/cluster-autoscaler/${module.this.id}-cluster" = "owned"
+    "k8s.io/cluster-autoscaler/enabled"                   = "true"
+    "kubernetes.io/cluster/${module.this.id}-cluster"     = "owned"
   }
 
   context = module.this.context
@@ -22,7 +25,7 @@ locals {
   # https://www.terraform.io/docs/providers/aws/guides/eks-getting-started.html#base-vpc-networking
 
   tags = merge(module.label.tags,
-  map("kubernetes.io/cluster/${module.label.id}", "shared"))
+  map("kubernetes.io/cluster/${module.label.id}-cluster", "shared"))
 
   # Unfortunately, most_recent (https://github.com/cloudposse/terraform-aws-eks-workers/blob/34a43c25624a6efb3ba5d2770a601d7cb3c0d391/main.tf#L141)
   # variable does not work as expected, if you are not going to use custom ami you should
@@ -40,7 +43,7 @@ locals {
   }
 
   k8s_service_account_namespace = "kube-system"
-  k8s_service_account_name      = "cluster-autoscaler-aws-cluster-autoscaler-chart"
+  k8s_service_account_name      = "cluster-autoscaler-aws-cluster-autoscaler"
 }
 
 module "eks_cluster" {
@@ -70,15 +73,14 @@ module "eks_cluster" {
 module "eks_workers" {
   depends_on = [module.eks_cluster]
   source     = "cloudposse/eks-workers/aws"
-  version = "0.19.2"
+  version    = "0.19.2"
 
 
   # for_each = tomap(var.eks_worker_groups)
   for_each = { for eks_worker_group in var.eks_worker_groups : eks_worker_group.name => eks_worker_group }
 
-  vpc_id     = var.vpc_id
-  subnet_ids = var.subnet_ids
-  # cluster_name = module.label.id
+  vpc_id                             = var.vpc_id
+  subnet_ids                         = var.subnet_ids
   cluster_name                       = data.null_data_source.wait_for_cluster_and_kubernetes_configmap.outputs["cluster_name"]
   cluster_endpoint                   = module.eks_cluster.eks_cluster_endpoint
   cluster_certificate_authority_data = module.eks_cluster.eks_cluster_certificate_authority_data
@@ -90,7 +92,13 @@ module "eks_workers" {
 
   tags = local.tags
 
-  bootstrap_extra_args = "--use-max-pods false"
+  autoscaling_group_tags = {
+    "k8s.io/cluster-autoscaler/${module.this.id}-cluster" = "owned"
+    "k8s.io/cluster-autoscaler/${module.this.id}"         = "owned"
+    "kubernetes.io/cluster/${module.this.id}-cluster"     = "owned"
+    "k8s.io/cluster-autoscaler/enabled"                   = "true"
+  }
+  # bootstrap_extra_args = "--use-max-pods false"
   # kubelet_extra_args   = "--node-labels=purpose=ci-worker"
 
   context = module.this.context
@@ -118,7 +126,7 @@ module "eks_workers" {
   ]
 
   # Auto-scaling policies and CloudWatch metric alarms
-  autoscaling_policies_enabled = var.autoscaling_policies_enabled
+  autoscaling_policies_enabled = var.eks_worker_group_autoscaling_policies_enabled
   # cpu_utilization_high_threshold_percent = var.cpu_utilization_high_threshold_percent
   # cpu_utilization_low_threshold_percent  = var.cpu_utilization_low_threshold_percent
 }
@@ -145,9 +153,10 @@ module "eks_node_group" {
   disk_size      = each.value.disk_size
 
   kubernetes_labels = var.kubernetes_labels
-  tags              = local.tags
+  # kubernetes_labels = local.tags
+  tags = local.tags
 
-  cluster_autoscaler_enabled = true
+  cluster_autoscaler_enabled = var.eks_node_group_autoscaling_enabled
   context                    = module.this.context
 }
 

diff --git a/outputs.tf b/outputs.tf
@@ -26,6 +26,21 @@ output "eks_cluster" {
   value       = module.eks_cluster
 }
 
+output "eks_cluster_identity_oidc_issuer" {
+  description = "The OIDC Identity issuer for the cluster"
+  value       = module.eks_cluster.eks_cluster_identity_oidc_issuer
+}
+
+# output "eks_cluster_identity_oidc_issuer" {
+#   description = "The OIDC Identity issuer for the cluster"
+#   value       = join("", aws_eks_cluster.default.*.identity.0.oidc.0.issuer)
+# }
+
+# output "cluster_oidc_issuer_url" {
+#   description = "The URL on the EKS cluster OIDC Issuer"
+#   value       = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc.0.issuer, [""]))[0]
+# }
+
 output "eks_cluster_node_groups" {
   value = module.eks_node_group
 }
@@ -37,4 +52,9 @@ output "eks_cluster_workers" {
 output "get_kubectl" {
   description = "Get your kubectl "
   value       = "aws eks get-token --cluster-name ${module.eks_cluster.eks_cluster_id}"
-}
+}
+
+output "cluster_autoscaler_iam_policy" {
+  description = ""
+  value       = aws_iam_policy.cluster_autoscaler
+}
diff --git a/variables.tf b/variables.tf
@@ -154,9 +154,9 @@ variable "cluster_encryption_config_resources" {
 # EKS Worker Groups 
 ####################################################################
 
-variable "autoscaling_policies_enabled" {
+variable "eks_worker_group_autoscaling_policies_enabled" {
   type        = bool
-  default     = true
+  default     = false
   description = "Whether to create `aws_autoscaling_policy` and `aws_cloudwatch_metric_alarm` resources to control Auto Scaling"
 }
 
@@ -206,3 +206,8 @@ variable "eks_node_groups" {
     }
   ]
 }
+
+variable "eks_node_group_autoscaling_enabled" {
+  type    = bool
+  default = false
+}