From 9cb886e1405290e93773afed5873959f34d97a46 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 30 Jan 2024 14:45:58 -0500 Subject: [PATCH] docs: Complete upgrade guide docs for migration and changes applied --- README.md | 2 +- docs/UPGRADE-20.0.md | 133 ++++++++++++++++++-- examples/eks_managed_node_group/README.md | 1 - examples/eks_managed_node_group/main.tf | 40 ++---- examples/fargate_profile/main.tf | 15 +-- examples/karpenter/README.md | 3 + examples/karpenter/main.tf | 23 ++-- examples/outposts/README.md | 7 ++ examples/outposts/main.tf | 3 +- examples/outposts/prerequisites/main.tf | 2 +- examples/self_managed_node_group/main.tf | 3 - main.tf | 8 +- modules/aws-auth/README.md | 38 +++++- modules/aws-auth/main.tf | 31 +---- modules/aws-auth/variables.tf | 18 --- modules/eks-managed-node-group/README.md | 3 - modules/eks-managed-node-group/main.tf | 15 --- modules/eks-managed-node-group/outputs.tf | 9 -- modules/eks-managed-node-group/variables.tf | 10 -- modules/fargate-profile/README.md | 3 - modules/fargate-profile/main.tf | 14 --- modules/fargate-profile/outputs.tf | 9 -- modules/fargate-profile/variables.tf | 10 -- modules/karpenter/README.md | 11 +- modules/karpenter/main.tf | 5 + modules/karpenter/migrations.tf | 13 +- modules/karpenter/variables.tf | 2 +- node_groups.tf | 6 - templates/aws_auth_cm.tpl | 37 ------ 29 files changed, 230 insertions(+), 244 deletions(-) delete mode 100644 templates/aws_auth_cm.tpl diff --git a/README.md b/README.md index 1d4ce814f9..1039fa6c82 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple ## License -Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-rds-aurora/tree/master/LICENSE) for full details. +Apache 2 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/LICENSE) for full details. ## Additional information for users from Russia and Belarus diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index 7d5fa44039..72bdd57699 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -19,6 +19,7 @@ To give users advanced notice and provide some future direction for this module, 1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider. 2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy. +3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum. ## Additional changes @@ -98,12 +99,6 @@ To give users advanced notice and provide some future direction for this module, - `access_entries` - `cloudwatch_log_group_class` - - EKS managed nodegroup - - `create_access_entry` - - - Fargate Profile - - `create_access_entry` - - Karpenter - `iam_policy_name` - `iam_policy_use_name_prefix` @@ -136,12 +131,6 @@ To give users advanced notice and provide some future direction for this module, - `access_entries` - - EKS managed nodegroup - - `access_entry_arn` - - - Fargate Profile - - `access_entry_arn` - - Karpenter - `node_access_entry_arn` @@ -150,4 +139,124 @@ To give users advanced notice and provide some future direction for this module, ## Upgrade Migrations +### Diff of Before (v19.21) vs After (v20.0) + +```diff + module "eks" { + source = "terraform-aws-modules/eks/aws" +- version = "~> 19.21" ++ version = "~> 20.0" + +# If you want to maintain the current default behavior of v19.x ++ kms_key_enable_default_policy = false + +- manage_aws_auth_configmap = true + +- aws_auth_roles = [ +- { +- rolearn = "arn:aws:iam::66666666666:role/role1" +- username = "role1" +- groups = ["custom-role-group"] +- }, +- ] + +- aws_auth_users = [ +- { +- userarn = "arn:aws:iam::66666666666:user/user1" +- username = "user1" +- groups = ["custom-users-group"] +- }, +- ] +} + ++ module "eks" { ++ source = "terraform-aws-modules/eks/aws//modules/aws-auth" ++ version = "~> 20.0" + ++ manage_aws_auth_configmap = true + ++ aws_auth_roles = [ ++ { ++ rolearn = "arn:aws:iam::66666666666:role/role1" ++ username = "role1" ++ groups = ["custom-role-group"] ++ }, ++ ] + ++ aws_auth_users = [ ++ { ++ userarn = "arn:aws:iam::66666666666:user/user1" ++ username = "user1" ++ groups = ["custom-users-group"] ++ }, ++ ] ++ } +``` + +### Karpenter Diff of Before (v19.21) vs After (v20.0) + +```diff + module "eks" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" +- version = "~> 19.21" ++ version = "~> 20.0" + +# If you wish to maintain the current default behavior of v19.x ++ enable_irsa = true ++ create_instance_profile = true + +# To avoid any resource re-creation ++ iam_role_name = "KarpenterIRSA-${module.eks.cluster_name}" ++ iam_role_description = "Karpenter IAM role for service account" ++ iam_policy_name = "KarpenterIRSA-${module.eks.cluster_name}" ++ iam_policy_description = "Karpenter IAM role for service account" +} +``` + ## Terraform State Moves + +#### ⚠️ Authentication Mode Changes ⚠️ + +Changing the `authentication_mode` is a one-way decision. See [announcement blog](https://aws.amazon.com/blogs/containers/a-deep-dive-into-simplified-amazon-eks-access-management-controls/) for further details: + +> Switching authentication modes on an existing cluster is a one-way operation. You can switch from CONFIG_MAP to API_AND_CONFIG_MAP. You can then switch from API_AND_CONFIG_MAP to API. You cannot revert these operations in the opposite direction. Meaning you cannot switch back to CONFIG_MAP or API_AND_CONFIG_MAP from API. And you cannot switch back to CONFIG_MAP from API_AND_CONFIG_MAP. + +### authentication_mode = "CONFIG_MAP" + +If using `authentication_mode = "CONFIG_MAP"`, before making any changes, you will first need to remove the configmap from the statefile to avoid any disruptions: + +```sh +terraform state rm 'module.eks.kubernetes_config_map_v1_data.aws_auth[0]' +terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if Terraform created the original configmap +``` + +Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). + +#### ⚠️ Node IAM Roles + +You will need to add entries for any IAM roles used by nodegroups and/or Fargate profiles - the module no longer handles this in the background on behalf of users. + +When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. + +### authentication_mode = "API_AND_CONFIG_MAP" + +When using `authentication_mode = "API_AND_CONFIG_MAP"` and there are entries that will remain in the configmap (entries that cannot be replaced by cluster access entry), you will first need to update the `authentication_mode` on the cluster to `"API_AND_CONFIG_MAP"`. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed nodegroups and Fargate profiles; users do not need to do anything additional for these roles. + +Once the `authentication_mode` has been updated, next you will need to remove the configmap from the statefile to avoid any disruptions: + +```sh +terraform state rm 'module.eks.kubernetes_config_map_v1_data.aws_auth[0]' +terraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]' # include if Terraform created the original configmap +``` + +#### ℹ️ Terraform 1.7+ users + +If you are using Terraform `v1.7+`, you can utilize the [`remove`](https://developer.hashicorp.com/terraform/language/resources/syntax#removing-resources) to facilitate both the removal of the configmap through code. You can create a fork/clone of the provided [migration module](https://github.com/clowdhaus/terraform-aws-eks-migrate-v19-to-v20) and add the `remove` blocks and apply those changes before proceeding. We do not want to force users onto the bleeding edge with this module, so we have not included `remove` support at this time. + +Once the configmap has been removed from the statefile, you can add the new `aws-auth` sub-module and copy the relevant definitions from the EKS module over to the new `aws-auth` sub-module definition (see before after diff above). When you apply the changes with the new sub-module, the configmap in the cluster will get updated with the contents provided in the sub-module definition, so please be sure all of the necessary entries are added before applying the changes. In the before/example above - the configmap would remove any entries for roles used by nodegroups and/or Fargate Profiles, but maintain the custom entries for users and roles passed into the module definition. + +### authentication_mode = "API" + +In order to switch to `API` only using cluster access entry, you first need to update the `authentication_mode` on the cluster to `API_AND_CONFIG_MAP` without modifying the `aws-auth` configmap. To help make this upgrade process easier, a copy of the changes defined in the [`v20.0.0`](https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2858) PR have been captured [here](https://github.com/clowdhaus/terraform-aws-eks-v20-migrate) but with the `aws-auth` components still provided in the module. This means you get the equivalent of the `v20.0.0` module, but it still includes support for the `aws-auth` configmap. You can follow the provided README on that interim migration module for the order of execution and return here once the `authentication_mode` has been updated to `"API_AND_CONFIG_MAP"`. Note - EKS automatically adds access entries for the roles used by EKS managed nodegroups and Fargate profiles; users do not need to do anything additional for these roles. + +Once the `authentication_mode` has been updated, you can update the `authentication_mode` on the cluster to `API` and remove the `aws-auth` configmap components. diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 82a4f644e1..103d133553 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -49,7 +49,6 @@ Note that this example may create resources which cost money. Run `terraform des | [eks\_managed\_node\_group](#module\_eks\_managed\_node\_group) | ../../modules/eks-managed-node-group | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | -| [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.0 | ## Resources diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 557c54fa69..0d17827fd8 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -7,7 +7,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" + cluster_version = "1.27" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -31,9 +31,6 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true - # Gives Terraform identity admin access to cluster - enable_cluster_creator_admin_permissions = true - # IPV6 cluster_ip_family = "ipv6" create_cni_ipv6_iam_policy = true @@ -46,9 +43,8 @@ module "eks" { most_recent = true } vpc-cni = { - most_recent = true - before_compute = true - service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + most_recent = true + before_compute = true configuration_values = jsonencode({ env = { # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html @@ -248,12 +244,6 @@ module "eks" { tags = local.tags } -module "disabled_eks_managed_node_group" { - source = "../../modules/eks-managed-node-group" - - create = false -} - module "disabled_eks" { source = "../.." @@ -294,6 +284,12 @@ module "eks_managed_node_group" { tags = merge(local.tags, { Separate = "eks-managed-node-group" }) } +module "disabled_eks_managed_node_group" { + source = "../../modules/eks-managed-node-group" + + create = false +} + ################################################################################ # Supporting Resources ################################################################################ @@ -333,24 +329,6 @@ module "vpc" { tags = local.tags } -module "vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 5.0" - - role_name_prefix = "VPC-CNI-IRSA" - attach_vpc_cni_policy = true - vpc_cni_enable_ipv6 = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node"] - } - } - - tags = local.tags -} - module "ebs_kms_key" { source = "terraform-aws-modules/kms/aws" version = "~> 2.1" diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index 479ed29c2a..0b3c6b46c2 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -30,9 +30,6 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true - # Gives Terraform identity admin access to cluster - enable_cluster_creator_admin_permissions = true - cluster_addons = { kube-proxy = {} vpc-cni = {} @@ -92,12 +89,6 @@ module "eks" { tags = local.tags } -module "disabled_fargate_profile" { - source = "../../modules/fargate-profile" - - create = false -} - ################################################################################ # Sub-Module Usage on Existing/Separate Cluster ################################################################################ @@ -116,6 +107,12 @@ module "fargate_profile" { tags = merge(local.tags, { Separate = "fargate-profile" }) } +module "disabled_fargate_profile" { + source = "../../modules/fargate-profile" + + create = false +} + ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 34d77e2b72..7b489f8780 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -41,6 +41,9 @@ kubectl delete node -l karpenter.sh/provisioner-name=default 2. Remove the resources created by Terraform ```bash +# Necessary to avoid removing Terraform's permissions too soon before its finished +# cleaning up the resources it deployed inside the clsuter +terraform state rm module.eks.aws_eks_access_entry.this["cluster_creator_admin"] || true terraform destroy ``` diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 06c9bcbf45..cd36785893 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -42,7 +42,7 @@ data "aws_ecrpublic_authorization_token" "token" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" + cluster_version = "1.28" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -66,7 +66,8 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true - # Gives Terraform identity admin access to cluster + # Gives Terraform identity admin access to cluster which will + # allow deploying resources (Karpenter) into the cluster enable_cluster_creator_admin_permissions = true cluster_addons = { @@ -149,6 +150,17 @@ module "karpenter" { tags = local.tags } +module "karpenter_disabled" { + source = "../../modules/karpenter" + + create = false +} + +################################################################################ +# Karpenter Helm chart & manifests +# Not required; just to demonstrate functionality of the sub-module +################################################################################ + resource "helm_release" "karpenter" { namespace = "karpenter" create_namespace = true @@ -158,6 +170,7 @@ resource "helm_release" "karpenter" { repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" version = "v0.33.1" + wait = false values = [ <<-EOT @@ -264,12 +277,6 @@ resource "kubectl_manifest" "karpenter_example_deployment" { ] } -module "karpenter_disabled" { - source = "../../modules/karpenter" - - create = false -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 0217da3c9e..47224216e4 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -36,6 +36,13 @@ $ terraform apply Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. +```bash +# Necessary to avoid removing Terraform's permissions too soon before its finished +# cleaning up the resources it deployed inside the clsuter +terraform state rm module.eks.aws_eks_access_entry.this["cluster_creator_admin"] || true +terraform destroy +``` + ## Requirements diff --git a/examples/outposts/main.tf b/examples/outposts/main.tf index 75dbcecfce..4b13f52465 100644 --- a/examples/outposts/main.tf +++ b/examples/outposts/main.tf @@ -29,7 +29,8 @@ module "eks" { cluster_endpoint_public_access = false # Not available on Outpost cluster_endpoint_private_access = true - # Gives Terraform identity admin access to cluster + # Gives Terraform identity admin access to cluster which will + # allow deploying resources (EBS storage class) into the cluster enable_cluster_creator_admin_permissions = true vpc_id = data.aws_vpc.this.id diff --git a/examples/outposts/prerequisites/main.tf b/examples/outposts/prerequisites/main.tf index 2193d6a610..66ab2a4e29 100644 --- a/examples/outposts/prerequisites/main.tf +++ b/examples/outposts/prerequisites/main.tf @@ -56,7 +56,7 @@ module "ssm_bastion_ec2" { rm terraform_${local.terraform_version}_linux_amd64.zip 2> /dev/null # Install kubectl - curl -LO https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl + curl -LO https://dl.k8s.io/release/v1.29.0/bin/linux/amd64/kubectl install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl # Remove default awscli which is v1 - we want latest v2 diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 350adae24a..dc125e1fbb 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -31,9 +31,6 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true - # Gives Terraform identity admin access to cluster - enable_cluster_creator_admin_permissions = true - cluster_addons = { coredns = { most_recent = true diff --git a/main.tf b/main.tf index b2ca174056..a934c0f38e 100644 --- a/main.tf +++ b/main.tf @@ -140,12 +140,12 @@ locals { # This replaces the one time logic from the EKS API with something that can be # better controlled by users through Terraform bootstrap_cluster_creator_admin_permissions = { - cluster_creator_admin = { + cluster_creator = { principal_arn = data.aws_iam_session_context.current.issuer_arn type = "STANDARD" policy_associations = { - clustrer_admin = { + admin = { policy_arn = "arn:${local.partition}:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" access_scope = { type = "cluster" @@ -189,7 +189,7 @@ locals { } resource "aws_eks_access_entry" "this" { - for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}-${v.pol_key}" => v if local.create } + for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } cluster_name = aws_eks_cluster.this[0].name kubernetes_groups = try(each.value.kubernetes_groups, []) @@ -201,7 +201,7 @@ resource "aws_eks_access_entry" "this" { } resource "aws_eks_access_policy_association" "this" { - for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}-${v.pol_key}" => v if local.create } + for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } access_scope { namespaces = try(each.value.association_access_scope_namespaces, []) diff --git a/modules/aws-auth/README.md b/modules/aws-auth/README.md index bb06503a75..d66aa9fca7 100644 --- a/modules/aws-auth/README.md +++ b/modules/aws-auth/README.md @@ -2,6 +2,41 @@ Configuration in this directory creates/updates the `aws-auth` ConfigMap. +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws//modules/aws-auth" + version = "~> 20.0" + + manage_aws_auth_configmap = true + + aws_auth_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + aws_auth_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + aws_auth_accounts = [ + "777777777777", + "888888888888", + ] +} +``` + ## Usage @@ -34,9 +69,6 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [aws\_auth\_accounts](#input\_aws\_auth\_accounts) | List of account maps to add to the aws-auth configmap | `list(any)` | `[]` | no | -| [aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns](#input\_aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns) | List of Fargate profile pod execution role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | -| [aws\_auth\_node\_iam\_role\_arns\_non\_windows](#input\_aws\_auth\_node\_iam\_role\_arns\_non\_windows) | List of non-Windows based node IAM role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | -| [aws\_auth\_node\_iam\_role\_arns\_windows](#input\_aws\_auth\_node\_iam\_role\_arns\_windows) | List of Windows based node IAM role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no | | [aws\_auth\_roles](#input\_aws\_auth\_roles) | List of role maps to add to the aws-auth configmap | `list(any)` | `[]` | no | | [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | | [create](#input\_create) | Controls if resources should be created (affects all resources) | `bool` | `true` | no | diff --git a/modules/aws-auth/main.tf b/modules/aws-auth/main.tf index b6243425eb..2f7e9694a7 100644 --- a/modules/aws-auth/main.tf +++ b/modules/aws-auth/main.tf @@ -5,36 +5,7 @@ locals { aws_auth_configmap_data = { - mapRoles = yamlencode(concat( - [for role_arn in var.aws_auth_node_iam_role_arns_non_windows : { - rolearn = role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }], - [for role_arn in var.aws_auth_node_iam_role_arns_windows : { - rolearn = role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "eks:kube-proxy-windows", - "system:bootstrappers", - "system:nodes", - ] - }], - # Fargate profile - [for role_arn in var.aws_auth_fargate_profile_pod_execution_role_arns : { - rolearn = role_arn - username = "system:node:{{SessionName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - "system:node-proxier", - ] - }], - var.aws_auth_roles - )) + mapRoles = yamlencode(var.aws_auth_roles) mapUsers = yamlencode(var.aws_auth_users) mapAccounts = yamlencode(var.aws_auth_accounts) } diff --git a/modules/aws-auth/variables.tf b/modules/aws-auth/variables.tf index a8dbe3038c..3aaeb023e3 100644 --- a/modules/aws-auth/variables.tf +++ b/modules/aws-auth/variables.tf @@ -20,24 +20,6 @@ variable "manage_aws_auth_configmap" { default = true } -variable "aws_auth_node_iam_role_arns_non_windows" { - description = "List of non-Windows based node IAM role ARNs to add to the aws-auth configmap" - type = list(string) - default = [] -} - -variable "aws_auth_node_iam_role_arns_windows" { - description = "List of Windows based node IAM role ARNs to add to the aws-auth configmap" - type = list(string) - default = [] -} - -variable "aws_auth_fargate_profile_pod_execution_role_arns" { - description = "List of Fargate profile pod execution role ARNs to add to the aws-auth configmap" - type = list(string) - default = [] -} - variable "aws_auth_roles" { description = "List of role maps to add to the aws-auth configmap" type = list(any) diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index 33eb3b0351..ebae013b92 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -83,7 +83,6 @@ module "eks_managed_node_group" { | Name | Type | |------|------| | [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | -| [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -113,7 +112,6 @@ module "eks_managed_node_group" { | [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | | [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | | [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | -| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the nodegroup | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | | [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | @@ -180,7 +178,6 @@ module "eks_managed_node_group" { | Name | Description | |------|-------------| -| [access\_entry\_arn](#output\_access\_entry\_arn) | Amazon Resource Name (ARN) of the Access Entry | | [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | | [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | | [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index fb8f2490ea..16ca010ae6 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -448,21 +448,6 @@ resource "aws_iam_role_policy_attachment" "additional" { role = aws_iam_role.this[0].name } -################################################################################ -# Access Entry -################################################################################ - -resource "aws_eks_access_entry" "this" { - count = var.create && var.create_access_entry ? 1 : 0 - - cluster_name = var.cluster_name - principal_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn - kubernetes_groups = null - type = var.platform == "windows" ? "EC2_WINDOWS" : "EC2_LINUX" - - tags = var.tags -} - ################################################################################ # Autoscaling Group Schedule ################################################################################ diff --git a/modules/eks-managed-node-group/outputs.tf b/modules/eks-managed-node-group/outputs.tf index aece4f75fb..a72f27b347 100644 --- a/modules/eks-managed-node-group/outputs.tf +++ b/modules/eks-managed-node-group/outputs.tf @@ -89,15 +89,6 @@ output "iam_role_unique_id" { value = try(aws_iam_role.this[0].unique_id, null) } -################################################################################ -# Access Entry -################################################################################ - -output "access_entry_arn" { - description = "Amazon Resource Name (ARN) of the Access Entry" - value = try(aws_eks_access_entry.this[0].access_entry_arn, null) -} - ################################################################################ # Additional ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index 07504ec677..ede9dc4c50 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -470,16 +470,6 @@ variable "iam_role_tags" { default = {} } -################################################################################ -# Access Entry -################################################################################ - -variable "create_access_entry" { - description = "Determines whether an access entry is created for the IAM role used by the nodegroup" - type = bool - default = true -} - ################################################################################ # Autoscaling Group Schedule ################################################################################ diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 2bfd5bb993..8656a6f191 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -45,7 +45,6 @@ No modules. | Name | Type | |------|------| -| [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | | [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -61,7 +60,6 @@ No modules. | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | | [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | -| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the Fargate Profile | `bool` | `true` | no | | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | @@ -82,7 +80,6 @@ No modules. | Name | Description | |------|-------------| -| [access\_entry\_arn](#output\_access\_entry\_arn) | Amazon Resource Name (ARN) of the Access Entry | | [fargate\_profile\_arn](#output\_fargate\_profile\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile | | [fargate\_profile\_id](#output\_fargate\_profile\_id) | EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`) | | [fargate\_profile\_pod\_execution\_role\_arn](#output\_fargate\_profile\_pod\_execution\_role\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile Pod execution role ARN | diff --git a/modules/fargate-profile/main.tf b/modules/fargate-profile/main.tf index 5085ab8abb..de9dd2d754 100644 --- a/modules/fargate-profile/main.tf +++ b/modules/fargate-profile/main.tf @@ -57,20 +57,6 @@ resource "aws_iam_role_policy_attachment" "additional" { role = aws_iam_role.this[0].name } -################################################################################ -# Access Entry -################################################################################ - -resource "aws_eks_access_entry" "this" { - count = var.create && var.create_access_entry ? 1 : 0 - - cluster_name = var.cluster_name - principal_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn - type = "FARGATE_LINUX" - - tags = var.tags -} - ################################################################################ # Fargate Profile ################################################################################ diff --git a/modules/fargate-profile/outputs.tf b/modules/fargate-profile/outputs.tf index d26e3a811b..96763bfb1f 100644 --- a/modules/fargate-profile/outputs.tf +++ b/modules/fargate-profile/outputs.tf @@ -17,15 +17,6 @@ output "iam_role_unique_id" { value = try(aws_iam_role.this[0].unique_id, null) } -################################################################################ -# Access Entry -################################################################################ - -output "access_entry_arn" { - description = "Amazon Resource Name (ARN) of the Access Entry" - value = try(aws_eks_access_entry.this[0].access_entry_arn, null) -} - ################################################################################ # Fargate Profile ################################################################################ diff --git a/modules/fargate-profile/variables.tf b/modules/fargate-profile/variables.tf index 4999dee84b..e22279dc6b 100644 --- a/modules/fargate-profile/variables.tf +++ b/modules/fargate-profile/variables.tf @@ -80,16 +80,6 @@ variable "iam_role_tags" { default = {} } -################################################################################ -# Access Entry -################################################################################ - -variable "create_access_entry" { - description = "Determines whether an access entry is created for the IAM role used by the Fargate Profile" - type = bool - default = true -} - ################################################################################ # Fargate Profile ################################################################################ diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 7a582d9558..029ca69816 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -10,7 +10,7 @@ In the following example, the Karpenter module will create: - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions - An access entry for the Node IAM role to allow nodes to join the cluster -- SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. +- SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. ```hcl module "eks" { @@ -40,9 +40,9 @@ module "karpenter" { In the following example, the Karpenter module will create: - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller -- SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. +- SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. -In this scenario, Karpenter will re-use an existing Node IAM role from the EKS Managed Nodegroup which already has the necessary access entry permissions: +In this scenario, Karpenter will re-use an existing Node IAM role from the EKS managed nodegroup which already has the necessary access entry permissions: ```hcl module "eks" { @@ -69,6 +69,9 @@ module "karpenter" { create_iam_role = false iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn + # Since the nodegroup role will already have an access entry + create_access_entry = false + tags = { Environment = "dev" Terraform = "true" @@ -123,7 +126,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [access\_entry\_type](#input\_access\_entry\_type) | Type of the access entry. `EC2_LINUX` or `EC2_WINDOWS`; defaults to `EC2_LINUX` | `string` | `"EC2_LINUX"` | no | +| [access\_entry\_type](#input\_access\_entry\_type) | Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX` | `string` | `"EC2_LINUX"` | no | | [ami\_id\_ssm\_parameter\_arns](#input\_ami\_id\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that Karpenter controller is allowed read access (for retrieving AMI IDs) | `list(string)` | `[]` | no | | [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true` | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index de84ee4cf9..dfd6042e34 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -587,6 +587,11 @@ resource "aws_eks_access_entry" "node" { type = var.access_entry_type tags = var.tags + + depends_on = [ + # If we try to add this too quickly, it fails. So .... we wait + aws_sqs_queue_policy.this, + ] } ################################################################################ diff --git a/modules/karpenter/migrations.tf b/modules/karpenter/migrations.tf index 9c10650dbb..3e7ca59a9b 100644 --- a/modules/karpenter/migrations.tf +++ b/modules/karpenter/migrations.tf @@ -1,5 +1,5 @@ ################################################################################ -# Migrations: v19.x -> v20.0 +# Migrations: v19.21 -> v20.0 ################################################################################ # Node IAM role @@ -43,3 +43,14 @@ moved { from = aws_iam_role_policy_attachment.irsa_additional to = aws_iam_role_policy_attachment.controller_additional } + +# Spelling correction +moved { + from = aws_cloudwatch_event_target.this["spot_interupt"] + to = aws_cloudwatch_event_target.this["spot_interrupt"] +} + +moved { + from = aws_cloudwatch_event_rule.this["spot_interupt"] + to = aws_cloudwatch_event_rule.this["spot_interrupt"] +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index d90e4d7366..3af82d4fc6 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -253,7 +253,7 @@ variable "create_access_entry" { } variable "access_entry_type" { - description = "Type of the access entry. `EC2_LINUX` or `EC2_WINDOWS`; defaults to `EC2_LINUX`" + description = "Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX`" type = string default = "EC2_LINUX" } diff --git a/node_groups.tf b/node_groups.tf index cf9c93807b..ddfc20fa09 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -259,9 +259,6 @@ module "fargate_profile" { # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.fargate_profile_defaults, "iam_role_additional_policies", {})) - # Access entry - create_access_entry = try(each.value.create_access_entry, var.fargate_profile_defaults.create_access_entry, true) - tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) } @@ -365,9 +362,6 @@ module "eks_managed_node_group" { # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) - # Access entry - create_access_entry = try(each.value.create_access_entry, var.eks_managed_node_group_defaults.create_access_entry, true) - # Autoscaling group schedule create_schedule = try(each.value.create_schedule, var.eks_managed_node_group_defaults.create_schedule, true) schedules = try(each.value.schedules, var.eks_managed_node_group_defaults.schedules, {}) diff --git a/templates/aws_auth_cm.tpl b/templates/aws_auth_cm.tpl deleted file mode 100644 index 73a898e966..0000000000 --- a/templates/aws_auth_cm.tpl +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aws-auth - namespace: kube-system -data: - mapRoles: | -%{ for role in eks_managed_role_arns ~} - - rolearn: ${role} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes -%{ endfor ~} -%{ for role in self_managed_role_arns ~} - - rolearn: ${role} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes -%{ endfor ~} -%{ for role in win32_self_managed_role_arns ~} - - rolearn: ${role} - username: system:node:{{EC2PrivateDNSName}} - groups: - - eks:kube-proxy-windows - - system:bootstrappers - - system:nodes -%{ endfor ~} -%{ for role in fargate_profile_pod_execution_role_arns ~} - - rolearn: ${role} - username: system:node:{{SessionName}} - groups: - - system:bootstrappers - - system:nodes - - system:node-proxier -%{ endfor ~}