diff --git a/README.md b/README.md index ff5fb79..c733f5f 100644 --- a/README.md +++ b/README.md @@ -17,16 +17,22 @@ The scope of this repo is to show all the AWS components needed to deploy a high # Table of Contents -* [Requirements](#requirements) -* [Infrastructure overview](#infrastructure-overview) -* [Before you start](#before-you-start) -* [Project setup](#project-setup) -* [AWS provider setup](#aws-provider-setup) -* [Pre flight checklist](#pre-flight-checklist) -* [Deploy](#deploy) -* [Deploy a sample stack](#deploy-a-sample-stack) -* [Clean up](#clean-up) -* [Todo](#todo) +- [Deploy Kubernetes on Amazon AWS](#deploy-kubernetes-on-amazon-aws) +- [Table of Contents](#table-of-contents) + - [Requirements](#requirements) + - [Before you start](#before-you-start) + - [Project setup](#project-setup) + - [AWS provider setup](#aws-provider-setup) + - [Pre flight checklist](#pre-flight-checklist) + - [Infrastructure overview](#infrastructure-overview) + - [Kubernetes setup](#kubernetes-setup) + - [Nginx ingress controller](#nginx-ingress-controller) + - [Cert manager](#cert-manager) + - [Deploy](#deploy) + - [Public LB check](#public-lb-check) + - [Deploy a sample stack](#deploy-a-sample-stack) + - [Clean up](#clean-up) + - [Todo](#todo) ## Requirements @@ -39,35 +45,8 @@ You need also: * one VPC with private and public subnets * one ssh key already uploaded on your AWS account -* one bastion host to reach all the private EC2 instances -For VPC and bastion host you can refer to [this](https://github.com/garutilorenzo/aws-terraform-examples) repository. - -## Infrastructure overview - -The final infrastructure will be made by: - -* two autoscaling group, one for the kubernetes master nodes and one for the worker nodes -* two launch template, used by the asg -* one internal load balancer (L4) that will route traffic to Kubernetes servers -* one external load balancer (L7) that will route traffic to Kubernetes workers -* one security group that will allow traffic from the VPC subnet CIDR on all the k8s ports (kube api, nginx ingress node port etc) -* one security group that will allow traffic from all the internet into the public load balancer (L7) on port 80 and 443 -* one S3 bucket, used to store the cluster join certificates -* one IAM role, used to allow all the EC2 instances in the cluster to write on the S3 bucket, used to share the join certificates -* one certificate used by the public LB, stored on AWS ACM. The certificate is a self signed certificate. - -![k8s infra](https://garutilorenzo.github.io/images/k8s-infra.png?) - -## Kubernetes setup - -The installation of K8s id done by [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/). In this installation [Containerd](https://containerd.io/) is used as CRI and [flannel](https://github.com/flannel-io/flannel) is used as CNI. - -You can optionally install [Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/) and [Longhorn](#https://longhorn.io/). - -To install Nginx ingress set the variable *install_nginx_ingress* to yes (default no). To install longhorn set the variable *install_longhorn* to yes (default no). **NOTE** if you don't install the nginx ingress, the public Load Balancer and the SSL certificate won't be deployed. - -In this installation is used a S3 bucket to store the join certificate/token. At the first startup of the instance, if the cluster does not exist, the S3 bucket is used to get the join certificates/token. +For VPC you can refer to [this](https://github.com/garutilorenzo/aws-terraform-examples) repository. ## Before you start @@ -82,13 +61,13 @@ git clone https://github.com/garutilorenzo/k8s-aws-terraform-cluster cd k8s-aws-terraform-cluster/example/ ``` -Now you have to edit the main.tf file and you have to create the terraform.tfvars file. For more detail see [AWS provider setup](#aws-provider-setup) and [Pre flight checklist](#pre-flight-checklist). +Now you have to edit the `main.tf` file and you have to create the `terraform.tfvars` file. For more detail see [AWS provider setup](#aws-provider-setup) and [Pre flight checklist](#pre-flight-checklist). Or if you prefer you can create an new empty directory in your workspace and create this three files: -* terraform.tfvars -* main.tf -* provider.tf +* `terraform.tfvars` +* `main.tf` +* `provider.tf` The main.tf file will look like: @@ -111,14 +90,11 @@ variable "AWS_REGION" { module "k8s-cluster" { ssk_key_pair_name = "" - uuid = "" environment = var.environment vpc_id = "" vpc_private_subnets = "" vpc_public_subnets = "" vpc_subnet_cidr = "" - PATH_TO_PUBLIC_LB_CERT = "" - PATH_TO_PUBLIC_LB_KEY = "" install_nginx_ingress = true source = "github.com/garutilorenzo/k8s-aws-terraform-cluster" } @@ -138,7 +114,7 @@ output "k8s_workers_private_ips" { For all the possible variables see [Pre flight checklist](#pre-flight-checklist) -The provider.tf will look like: +The `provider.tf` will look like: ``` provider "aws" { @@ -148,7 +124,7 @@ provider "aws" { } ``` -The terraform.tfvars will look like: +The `terraform.tfvars` will look like: ``` AWS_ACCESS_KEY = "xxxxxxxxxxxxxxxxx" @@ -189,73 +165,10 @@ rerun this command to reinitialize your working directory. If you forget, other commands will detect it and remind you to do so if necessary. ``` -### Generate self signed SSL certificate for the public LB (L7) - -**NOTE** If you already own a valid certificate skip this step and set the correct values for the variables: PATH_TO_PUBLIC_LB_CERT and PATH_TO_PUBLIC_LB_KEY - -We need to generate the certificates (sel signed) for our public load balancer (Layer 7). To do this we need *openssl*, open a terminal and follow this step: - -Generate the key: - -``` -openssl genrsa 2048 > privatekey.pem -Generating RSA private key, 2048 bit long modulus (2 primes) -.......+++++ -...............+++++ -e is 65537 (0x010001) -``` - -Generate the a new certificate request: - -``` -openssl req -new -key privatekey.pem -out csr.pem -You are about to be asked to enter information that will be incorporated -into your certificate request. -What you are about to enter is what is called a Distinguished Name or a DN. -There are quite a few fields but you can leave some blank -For some fields there will be a default value, -If you enter '.', the field will be left blank. ------ -Country Name (2 letter code) [AU]:IT -State or Province Name (full name) [Some-State]:Italy -Locality Name (eg, city) []:Brescia -Organization Name (eg, company) [Internet Widgits Pty Ltd]:GL Ltd -Organizational Unit Name (eg, section) []:IT -Common Name (e.g. server FQDN or YOUR name) []:testlb.domainexample.com -Email Address []:email@you.com - -Please enter the following 'extra' attributes -to be sent with your certificate request -A challenge password []: -An optional company name []: -``` - -Generate the public CRT: - -``` -openssl x509 -req -days 365 -in csr.pem -signkey privatekey.pem -out public.crt -Signature ok -subject=C = IT, ST = Italy, L = Brescia, O = GL Ltd, OU = IT, CN = testlb.domainexample.com, emailAddress = email@you.com -Getting Private key -``` - -This is the final result: - -``` -ls - -csr.pem privatekey.pem public.crt -``` - -Now set the variables: - -* PATH_TO_PUBLIC_LB_CERT: ~/full_path/public.crt -* PATH_TO_PUBLIC_LB_KEY: ~/full_path/privatekey.pem - ## AWS provider setup Follow the prerequisites step on [this](https://learn.hashicorp.com/tutorials/terraform/aws-build?in=terraform/aws-get-started) link. -In your workspace folder or in the examples directory of this repo create a file named terraform.tfvars: +In your workspace folder or in the examples directory of this repo create a file named `terraform.tfvars`: ``` AWS_ACCESS_KEY = "xxxxxxxxxxxxxxxxx" @@ -264,35 +177,29 @@ AWS_SECRET_KEY = "xxxxxxxxxxxxxxxxx" ## Pre flight checklist -Once you have created the terraform.tfvars file edit the main.tf file (always in the example/ directory) and set the following variables: +Once you have created the `terraform.tfvars` file edit the `main.tf` file (always in the example/ directory) and set the following variables: | Var | Required | Desc | | ------- | ------- | ----------- | | `region` | `yes` | set the correct OCI region based on your needs | | `environment` | `yes` | Current work environment (Example: staging/dev/prod). This value is used for tag all the deployed resources | -| `uuid` | `yes` | UUID used to tag all resources | | `ssk_key_pair_name` | `yes` | Name of the ssh key to use | +| `my_public_ip_cidr` | `yes` | your public ip in cidr format (Example: 195.102.xxx.xxx/32) | | `vpc_id` | `yes` | ID of the VPC to use. You can find your vpc_id in your AWS console (Example: vpc-xxxxx) | | `vpc_private_subnets` | `yes` | List of private subnets to use. This subnets are used for the public LB You can find the list of your vpc subnets in your AWS console (Example: subnet-xxxxxx) | | `vpc_public_subnets` | `yes` | List of public subnets to use. This subnets are used for the EC2 instances and the private LB. You can find the list of your vpc subnets in your AWS console (Example: subnet-xxxxxx) | | `vpc_subnet_cidr` | `yes` | Your subnet CIDR. You can find the VPC subnet CIDR in your AWS console (Example: 172.31.0.0/16) | -| `PATH_TO_PUBLIC_LB_CERT` | `yes` | Path to the public LB certificate. See [how to](#generate-self-signed-ssl-certificate-for-the-public-lb-l7) generate the certificate | -| `PATH_TO_PUBLIC_LB_KEY` | `yes` | Path to the public LB key. See [how to](#generate-self-signed-ssl-certificate-for-the-public-lb-l7) generate the key | +| `common_prefix` | `no` | Prefix used in all resource names/tags. Default: k8s | | `ec2_associate_public_ip_address` | `no` | Assign or not a pulic ip to the EC2 instances. Default: false | -| `s3_bucket_name` | `no` | S3 bucket name used for sharing the kubernetes token used for joining the cluster. Default: my-very-secure-k8s-bucket | | `instance_profile_name` | `no` | Instance profile name. Default: K8sInstanceProfile | -| `iam_role_name` | `no` | IAM role name. Default: K8sIamRole | | `ami` | `no` | Ami image name. Default: ami-0a2616929f1e63d91, ubuntu 20.04 | | `default_instance_type` | `no` | Default instance type used by the Launch template. Default: t3.large | | `instance_types` | `no` | Array of instances used by the ASG. Dfault: { asg_instance_type_1 = "t3.large", asg_instance_type_3 = "m4.large", asg_instance_type_4 = "t3a.large" } | -| `k8s_master_template_prefix` | `no` | Template prefix for the master instances. Default: k8s_master_tpl | -| `k8s_worker_template_prefix` | `no` | Template prefix for the worker instances. Default: k8s_worker_tpl | | `k8s_version` | `no` | Kubernetes version to install | | `k8s_pod_subnet` | `no` | Kubernetes pod subnet managed by the CNI (Flannel). Default: 10.244.0.0/16 | | `k8s_service_subnet` | `no` | Kubernetes pod service managed by the CNI (Flannel). Default: 10.96.0.0/12 | | `k8s_dns_domain` | `no` | Internal kubernetes DNS domain. Default: cluster.local | | `kube_api_port` | `no` | Kubernetes api port. Default: 6443 | -| `k8s_internal_lb_name` | `no` | Internal load balancer name. Default: k8s-server-tcp-lb | | `k8s_server_desired_capacity` | `no` | Desired number of k8s servers. Default 3 | | `k8s_server_min_capacity` | `no` | Min number of k8s servers: Default 4 | | `k8s_server_max_capacity` | `no` | Max number of k8s servers: Default 3 | @@ -300,14 +207,113 @@ Once you have created the terraform.tfvars file edit the main.tf file (always in | `k8s_worker_min_capacity` | `no` | Min number of k8s workers: Default 4 | | `k8s_worker_max_capacity` | `no` | Max number of k8s workers: Default 3 | | `cluster_name` | `no` | Kubernetes cluster name. Default: k8s-cluster | -| `install_longhorn` | `no` | Install or not longhorn. Default: false | -| `longhorn_release` | `no` | longhorn release. Default: v1.2.3 | | `install_nginx_ingress` | `no` | Install or not nginx ingress controller. Default: false | -| `k8s_ext_lb_name` | `no` | External load balancer name. Default: k8s-ext-lb | +| `nginx_ingress_release` | `no` | Nginx ingress release to install. Default: v1.8.1| +| `install_certmanager` | `no` | Boolean value, install [cert manager](https://cert-manager.io/) "Cloud native certificate management". Default: true | +| `certmanager_email_address` | `no` | Email address used for signing https certificates. Defaul: changeme@example.com | +| `certmanager_release` | `no` | Cert manager release. Default: v1.12.2 | +| `efs_persistent_storage` | `no` | Deploy EFS for persistent sotrage | +| `efs_csi_driver_release` | `no` | EFS CSI driver Release: v1.5.8 | | `extlb_listener_http_port` | `no` | HTTP nodeport where nginx ingress controller will listen. Default: 30080 | | `extlb_listener_https_port` | `no` | HTTPS nodeport where nginx ingress controller will listen. Default 30443 | | `extlb_http_port` | `no` | External LB HTTP listen port. Default: 80 | | `extlb_https_port` | `no` | External LB HTTPS listen port. Default 443 | +| `expose_kubeapi` | `no` | Boolean value, default false. Expose or not the kubeapi server to the internet. Access is granted only from *my_public_ip_cidr* for security reasons. | + +## Infrastructure overview + +The final infrastructure will be made by: + +* two autoscaling group, one for the kubernetes master nodes and one for the worker nodes +* two launch template, used by the asg +* one internal load balancer (L4) that will route traffic to Kubernetes servers +* one external load balancer (L4) that will route traffic to Kubernetes workers +* one security group that will allow traffic from the VPC subnet CIDR on all the k8s ports (kube api, nginx ingress node port etc) +* one security group that will allow traffic from all the internet into the public load balancer (L4) on port 80 and 443 +* four secrets that will store k8s join tokens + +Optional resources: + +* EFS storage to persist data + +## Kubernetes setup + +The installation of K8s id done by [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/). In this installation [Containerd](https://containerd.io/) is used as CRI and [flannel](https://github.com/flannel-io/flannel) is used as CNI. + +You can optionally install [Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/). + +To install Nginx ingress set the variable *install_nginx_ingress* to yes (default no). + +### Nginx ingress controller + +You can optionally install [Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/) To enable the Nginx deployment set `install_nginx_ingress` variable to `true`. + +The installation is the [bare metal](https://kubernetes.github.io/ingress-nginx/deploy/#bare-metal-clusters) installation, the ingress controller then is exposed via a NodePort Service. + +```yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + ports: + - appProtocol: http + name: http + port: 80 + protocol: TCP + targetPort: http + nodePort: ${extlb_listener_http_port} + - appProtocol: https + name: https + port: 443 + protocol: TCP + targetPort: https + nodePort: ${extlb_listener_https_port} + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: NodePort +``` + +To get the real ip address of the clients using a public L4 load balancer we need to use the proxy protocol feature of nginx ingress controller: + +```yaml +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + enable-real-ip: "true" + proxy-real-ip-cidr: "0.0.0.0/0" + proxy-body-size: "20m" + use-proxy-protocol: "true" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: ${nginx_ingress_release} + name: ingress-nginx-controller + namespace: ingress-nginx +``` + +**NOTE** to use nginx ingress controller with the proxy protocol enabled, an external nginx instance is used as proxy (since OCI LB doesn't support proxy protocol at the moment). Nginx will be installed on each worker node and the configuation of nginx will: + +* listen in proxy protocol mode +* forward the traffic from port `80` to `extlb_http_port` (default to `30080`) on any server of the cluster +* forward the traffic from port `443` to `extlb_https_port` (default to `30443`) on any server of the cluster + +This is the final result: + +Client -> Public L4 LB (with proxy protocol enabled) -> nginx ingress (with proxy protocol enabled) -> k8s service -> pod(s) + +### Cert-manager + +[cert-manager](https://cert-manager.io/docs/) is used to issue certificates from a variety of supported source. To use cert-manager take a look at [nginx-ingress-cert-manager.yml](https://github.com/garutilorenzo/k3s-oci-cluster/blob/master/deployments/nginx/nginx-ingress-cert-manager.yml) and [nginx-configmap-cert-manager.yml](https://github.com/garutilorenzo/k3s-oci-cluster/blob/master/deployments/nginx/nginx-configmap-cert-manager.yml) example. To use cert-manager and get the certificate you **need** set on your DNS configuration the public ip address of the load balancer. ## Deploy @@ -318,37 +324,33 @@ terraform plan ... ... - + name = "k8s-sg" - + name_prefix = (known after apply) - + owner_id = (known after apply) - + revoke_rules_on_delete = false - + tags = { - + "Name" = "sg-k8s-cluster-staging" - + "environment" = "staging" - + "provisioner" = "terraform" - + "scope" = "k8s-cluster" - + "uuid" = "xxxxx-xxxxx-xxxx-xxxxxx-xxxxxx" - } - + tags_all = { - + "Name" = "sg-k8s-cluster-staging" - + "environment" = "staging" - + "provisioner" = "terraform" - + "scope" = "k8s-cluster" - + "uuid" = "xxxxx-xxxxx-xxxx-xxxxxx-xxxxxx" - } - + vpc_id = "vpc-xxxxxx" - } - -Plan: 25 to add, 0 to change, 0 to destroy. +Plan: 73 to add, 0 to change, 0 to destroy. Changes to Outputs: - + k8s_dns_name = (known after apply) - + k8s_server_private_ips = [ + + k8s_dns_name = [ + + (known after apply), + ] + ~ k8s_server_private_ips = [ + - [], + + (known after apply), + ] + ~ k8s_workers_private_ips = [ + - [], + + (known after apply), + ] + + private_subnets_ids = [ + + (known after apply), + + (known after apply), + (known after apply), ] - + k8s_workers_private_ips = [ + + public_subnets_ids = [ + + (known after apply), + + (known after apply), + (known after apply), ] + + vpc_id = (known after apply) + +─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── Note: You didn't use the -out option to save this plan, so Terraform can't guarantee to take exactly these actions if you run "terraform apply" now. ``` @@ -360,26 +362,31 @@ terraform apply ... - + tags_all = { - + "Name" = "sg-k8s-cluster-staging" - + "environment" = "staging" - + "provisioner" = "terraform" - + "scope" = "k8s-cluster" - + "uuid" = "xxxxx-xxxxx-xxxx-xxxxxx-xxxxxx" - } - + vpc_id = "vpc-xxxxxxxx" - } - -Plan: 25 to add, 0 to change, 0 to destroy. +Plan: 73 to add, 0 to change, 0 to destroy. Changes to Outputs: - + k8s_dns_name = (known after apply) - + k8s_server_private_ips = [ + + k8s_dns_name = [ + + (known after apply), + ] + ~ k8s_server_private_ips = [ + - [], + + (known after apply), + ] + ~ k8s_workers_private_ips = [ + - [], + + (known after apply), + ] + + private_subnets_ids = [ + + (known after apply), + + (known after apply), + (known after apply), ] - + k8s_workers_private_ips = [ + + public_subnets_ids = [ + + (known after apply), + + (known after apply), + (known after apply), ] + + vpc_id = (known after apply) Do you want to perform these actions? Terraform will perform the actions described above. @@ -390,7 +397,7 @@ Do you want to perform these actions? ... ... -Apply complete! Resources: 25 added, 0 changed, 0 destroyed. +Apply complete! Resources: 73 added, 0 changed, 0 destroyed. Outputs: @@ -409,30 +416,21 @@ k8s_workers_private_ips = [ "172.x.x.x", ]), ] +private_subnets_ids = [ + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", +] +public_subnets_ids = [ + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", + "subnet-xxxxxxxxxxxxxxxxx", +] +vpc_id = "vpc-xxxxxxxxxxxxxxxxx" ``` -Now on one master node you can check the status of the cluster with: +Now on one master node (connect via AWS SSM) you can check the status of the cluster with: ``` -ssh -j bastion@ ubuntu@172.x.x.x - -Welcome to Ubuntu 20.04.4 LTS (GNU/Linux 5.13.0-1021-aws x86_64) - - * Documentation: https://help.ubuntu.com - * Management: https://landscape.canonical.com - * Support: https://ubuntu.com/advantage - - System information as of Wed Apr 13 12:41:52 UTC 2022 - - System load: 0.52 Processes: 157 - Usage of /: 17.8% of 19.32GB Users logged in: 0 - Memory usage: 11% IPv4 address for cni0: 10.244.0.1 - Swap usage: 0% IPv4 address for ens3: 172.68.4.237 - - -0 updates can be applied immediately. - - -Last login: Wed Apr 13 12:40:32 2022 from 172.68.0.6 ubuntu@i-04d089ed896cfafe1:~$ sudo su - root@i-04d089ed896cfafe1:~# kubectl get nodes @@ -446,73 +444,62 @@ i-0cb1e2e7784768b22 Ready 3m57s v1.23.5 root@i-04d089ed896cfafe1:~# kubectl get ns NAME STATUS AGE -default Active 5m18s -ingress-nginx Active 111s # <- ingress controller ns -kube-node-lease Active 5m19s -kube-public Active 5m19s -kube-system Active 5m19s -longhorn-system Active 109s # <- longhorn ns +cert-manager Active 85s +default Active 4m55s +ingress-nginx Active 87s # <- ingress controller ns +kube-flannel Active 4m32s +kube-node-lease Active 4m55s +kube-public Active 4m56s +kube-system Active 4m56s root@i-04d089ed896cfafe1:~# kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -ingress-nginx ingress-nginx-admission-create-v2fpx 0/1 Completed 0 2m33s -ingress-nginx ingress-nginx-admission-patch-54d9f 0/1 Completed 0 2m33s -ingress-nginx ingress-nginx-controller-7fc8d55869-cxv87 1/1 Running 0 2m33s -kube-system coredns-64897985d-8cg8g 1/1 Running 0 5m46s -kube-system coredns-64897985d-9v2r8 1/1 Running 0 5m46s -kube-system etcd-i-0033b408f7a1d55f3 1/1 Running 0 4m33s -kube-system etcd-i-04d089ed896cfafe1 1/1 Running 0 5m42s -kube-system etcd-i-09b23242f40eabcca 1/1 Running 0 5m -kube-system kube-apiserver-i-0033b408f7a1d55f3 1/1 Running 1 (4m30s ago) 4m30s -kube-system kube-apiserver-i-04d089ed896cfafe1 1/1 Running 0 5m46s -kube-system kube-apiserver-i-09b23242f40eabcca 1/1 Running 0 5m1s -kube-system kube-controller-manager-i-0033b408f7a1d55f3 1/1 Running 0 4m36s -kube-system kube-controller-manager-i-04d089ed896cfafe1 1/1 Running 1 (4m50s ago) 5m49s -kube-system kube-controller-manager-i-09b23242f40eabcca 1/1 Running 0 5m1s -kube-system kube-flannel-ds-7c65s 1/1 Running 0 5m2s -kube-system kube-flannel-ds-bb842 1/1 Running 0 4m10s -kube-system kube-flannel-ds-q27gs 1/1 Running 0 5m21s -kube-system kube-flannel-ds-sww7p 1/1 Running 0 5m3s -kube-system kube-flannel-ds-z8h5p 1/1 Running 0 5m38s -kube-system kube-flannel-ds-zrwdq 1/1 Running 0 5m22s -kube-system kube-proxy-6rbks 1/1 Running 0 5m2s -kube-system kube-proxy-9npgg 1/1 Running 0 5m21s -kube-system kube-proxy-px6br 1/1 Running 0 5m3s -kube-system kube-proxy-q9889 1/1 Running 0 4m10s -kube-system kube-proxy-s5qnv 1/1 Running 0 5m22s -kube-system kube-proxy-tng4x 1/1 Running 0 5m46s -kube-system kube-scheduler-i-0033b408f7a1d55f3 1/1 Running 0 4m27s -kube-system kube-scheduler-i-04d089ed896cfafe1 1/1 Running 1 (4m50s ago) 5m58s -kube-system kube-scheduler-i-09b23242f40eabcca 1/1 Running 0 5m1s -longhorn-system csi-attacher-6454556647-767p2 1/1 Running 0 115s -longhorn-system csi-attacher-6454556647-hz8lj 1/1 Running 0 115s -longhorn-system csi-attacher-6454556647-z5ftg 1/1 Running 0 115s -longhorn-system csi-provisioner-869bdc4b79-2v4wx 1/1 Running 0 115s -longhorn-system csi-provisioner-869bdc4b79-4xcv4 1/1 Running 0 114s -longhorn-system csi-provisioner-869bdc4b79-9q95d 1/1 Running 0 114s -longhorn-system csi-resizer-6d8cf5f99f-dwdrq 1/1 Running 0 114s -longhorn-system csi-resizer-6d8cf5f99f-klvcr 1/1 Running 0 114s -longhorn-system csi-resizer-6d8cf5f99f-ptpzb 1/1 Running 0 114s -longhorn-system csi-snapshotter-588457fcdf-dlkdq 1/1 Running 0 113s -longhorn-system csi-snapshotter-588457fcdf-p2c7c 1/1 Running 0 113s -longhorn-system csi-snapshotter-588457fcdf-p5smn 1/1 Running 0 113s -longhorn-system engine-image-ei-fa2dfbf0-bkwhx 1/1 Running 0 2m7s -longhorn-system engine-image-ei-fa2dfbf0-cqq9n 1/1 Running 0 2m8s -longhorn-system engine-image-ei-fa2dfbf0-lhjjc 1/1 Running 0 2m7s -longhorn-system instance-manager-e-542b1382 1/1 Running 0 119s -longhorn-system instance-manager-e-a5e124bb 1/1 Running 0 2m4s -longhorn-system instance-manager-e-acb2a517 1/1 Running 0 2m7s -longhorn-system instance-manager-r-11ab6af6 1/1 Running 0 119s -longhorn-system instance-manager-r-5b82fba2 1/1 Running 0 2m4s -longhorn-system instance-manager-r-c2561fa0 1/1 Running 0 2m6s -longhorn-system longhorn-csi-plugin-4br28 2/2 Running 0 113s -longhorn-system longhorn-csi-plugin-8gdxf 2/2 Running 0 113s -longhorn-system longhorn-csi-plugin-wc6tt 2/2 Running 0 113s -longhorn-system longhorn-driver-deployer-7dddcdd5bb-zjh4k 1/1 Running 0 2m31s -longhorn-system longhorn-manager-cbsh7 1/1 Running 0 2m31s -longhorn-system longhorn-manager-d2t75 1/1 Running 1 (2m9s ago) 2m31s -longhorn-system longhorn-manager-xqlfv 1/1 Running 1 (2m9s ago) 2m31s -longhorn-system longhorn-ui-7648d6cd69-tc6b9 1/1 Running 0 2m31s +NAMESPACE NAME READY STATUS RESTARTS AGE +cert-manager cert-manager-66d9545484-h4d9h 1/1 Running 0 47s +cert-manager cert-manager-cainjector-7d8b6bd6fb-zl7sg 1/1 Running 0 47s +cert-manager cert-manager-webhook-669b96dcfd-b5pgk 1/1 Running 0 47s +ingress-nginx ingress-nginx-admission-create-g62rk 0/1 Completed 0 50s +ingress-nginx ingress-nginx-admission-patch-n9tc5 0/1 Completed 0 50s +ingress-nginx ingress-nginx-controller-5c778bffff-bmk2c 1/1 Running 0 50s +kube-flannel kube-flannel-ds-5fvx9 1/1 Running 0 3m45s +kube-flannel kube-flannel-ds-bvqkc 1/1 Running 1 (3m13s ago) 3m35s +kube-flannel kube-flannel-ds-hgxtn 1/1 Running 1 (111s ago) 2m40s +kube-flannel kube-flannel-ds-kp6tl 1/1 Running 0 3m27s +kube-flannel kube-flannel-ds-nvbbg 1/1 Running 0 3m55s +kube-flannel kube-flannel-ds-rhsqq 1/1 Running 0 2m42s +kube-system aws-node-termination-handler-478lj 1/1 Running 0 26s +kube-system aws-node-termination-handler-5bk96 1/1 Running 0 26s +kube-system aws-node-termination-handler-bkzrf 1/1 Running 0 26s +kube-system aws-node-termination-handler-cx5ps 1/1 Running 0 26s +kube-system aws-node-termination-handler-dfr44 1/1 Running 0 26s +kube-system aws-node-termination-handler-vcq7z 1/1 Running 0 26s +kube-system coredns-5d78c9869d-n7jcq 1/1 Running 0 4m1s +kube-system coredns-5d78c9869d-w9k5j 1/1 Running 0 4m1s +kube-system efs-csi-controller-74695cd876-66bw5 3/3 Running 0 28s +kube-system efs-csi-controller-74695cd876-hl9g7 3/3 Running 0 28s +kube-system efs-csi-node-7wgff 3/3 Running 0 27s +kube-system efs-csi-node-9v4nv 3/3 Running 0 27s +kube-system efs-csi-node-mjz2r 3/3 Running 0 27s +kube-system efs-csi-node-n4npv 3/3 Running 0 27s +kube-system efs-csi-node-pmpnc 3/3 Running 0 27s +kube-system efs-csi-node-s4prq 3/3 Running 0 27s +kube-system etcd-i-012c258d537d5ec2f 1/1 Running 0 4m4s +kube-system etcd-i-018fb1214f9fe07fe 1/1 Running 0 3m7s +kube-system etcd-i-0f73570d6dddb6d0b 1/1 Running 0 3m27s +kube-system kube-apiserver-i-012c258d537d5ec2f 1/1 Running 0 4m6s +kube-system kube-apiserver-i-018fb1214f9fe07fe 1/1 Running 1 (3m4s ago) 3m4s +kube-system kube-apiserver-i-0f73570d6dddb6d0b 1/1 Running 0 3m26s +kube-system kube-controller-manager-i-012c258d537d5ec2f 1/1 Running 1 (3m15s ago) 4m7s +kube-system kube-controller-manager-i-018fb1214f9fe07fe 1/1 Running 0 2m9s +kube-system kube-controller-manager-i-0f73570d6dddb6d0b 1/1 Running 0 3m26s +kube-system kube-proxy-4lwgv 1/1 Running 0 2m40s +kube-system kube-proxy-9hgtr 1/1 Running 0 3m27s +kube-system kube-proxy-d6zzp 1/1 Running 0 4m1s +kube-system kube-proxy-jwb8x 1/1 Running 0 3m35s +kube-system kube-proxy-q2ctc 1/1 Running 0 2m42s +kube-system kube-proxy-sgn7r 1/1 Running 0 3m45s +kube-system kube-scheduler-i-012c258d537d5ec2f 1/1 Running 1 (3m12s ago) 4m6s +kube-system kube-scheduler-i-018fb1214f9fe07fe 1/1 Running 0 3m1s +kube-system kube-scheduler-i-0f73570d6dddb6d0b 1/1 Running 0 3m26s ``` #### Public LB check @@ -576,81 +563,10 @@ curl -k -v https://k8s-ext-.elb.amazonaws.com/ ## Deploy a sample stack -We use the same stack used in [this](https://github.com/garutilorenzo/k3s-oci-cluster) repository. -This stack **need** longhorn and nginx ingress. - -To test all the components of the cluster we can deploy a sample stack. The stack is composed by the following components: - -* MariaDB -* Nginx -* Wordpress - -Each component is made by: one deployment and one service. -Wordpress and nginx share the same persistent volume (ReadWriteMany with longhorn storage class). The nginx configuration is stored in four ConfigMaps and the nginx service is exposed by the nginx ingress controller. - -Deploy the resources with: - -``` -kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k3s-oci-cluster/master/deployments/mariadb/all-resources.yml -kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k3s-oci-cluster/master/deployments/nginx/all-resources.yml -kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k3s-oci-cluster/master/deployments/wordpress/all-resources.yml -``` - -**NOTE** to install WP and reach the *wp-admin* path you have to edit the nginx deployment and change this line: - -```yaml - env: - - name: SECURE_SUBNET - value: 8.8.8.8/32 # change-me -``` - -and set your public ip address. - -To check the status: - -``` -root@i-04d089ed896cfafe1:~# kubectl get pods -o wide -NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -mariadb-6cbf998bd6-s98nh 1/1 Running 0 2m21s 10.244.2.13 i-072bf7de2e94e6f2d -nginx-68b4dfbcb6-s6zfh 1/1 Running 0 19s 10.244.1.12 i-0121c2149821379cc -wordpress-558948b576-jgvm2 1/1 Running 0 71s 10.244.3.14 i-0cb1e2e7784768b22 - -root@i-04d089ed896cfafe1:~# kubectl get deployments -NAME READY UP-TO-DATE AVAILABLE AGE -mariadb 1/1 1 1 2m32s -nginx 1/1 1 1 30s -wordpress 1/1 1 1 82s - -root@i-04d089ed896cfafe1:~# kubectl get svc -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -kubernetes ClusterIP 10.96.0.1 443/TCP 14m -mariadb-svc ClusterIP 10.108.78.60 3306/TCP 2m43s -nginx-svc ClusterIP 10.103.145.57 80/TCP 41s -wordpress-svc ClusterIP 10.103.49.246 9000/TCP 93s -``` - -Now you are ready to setup WP, open the LB public ip and follow the wizard. **NOTE** nginx and the Kubernetes Ingress rule are configured without virthual host/server name. - -![k8s wp install](https://garutilorenzo.github.io/images/k8s-wp.png?) - -To clean the deployed resources: - -``` -kubectl delete -f https://raw.githubusercontent.com/garutilorenzo/k3s-oci-cluster/master/deployments/mariadb/all-resources.yml -kubectl delete -f https://raw.githubusercontent.com/garutilorenzo/k3s-oci-cluster/master/deployments/nginx/all-resources.yml -kubectl delete -f https://raw.githubusercontent.com/garutilorenzo/k3s-oci-cluster/master/deployments/wordpress/all-resources.yml -``` +[Deploy ECK](deployments/) on Kubernetes ## Clean up -Before destroy all the infrastructure **DELETE** all the object in the S3 bucket. - ``` terraform destroy -``` - -## TODO - -* Extend the IAM role for the cluster autoscaler -* Install the node termination handler for the EC2 spot instances -* Auto update the certificate/token on the S3 bucket, at the moment the certificate i generated only once. +``` \ No newline at end of file diff --git a/asg.tf b/asg.tf index 9e27dcf..e0652b5 100644 --- a/asg.tf +++ b/asg.tf @@ -1,5 +1,5 @@ resource "aws_autoscaling_group" "k8s_servers_asg" { - name = "k8s_servers" + name = "${var.common_prefix}-servers-asg-${var.environment}" wait_for_capacity_timeout = "5m" vpc_zone_identifier = var.vpc_private_subnets @@ -28,7 +28,6 @@ resource "aws_autoscaling_group" "k8s_servers_asg" { weighted_capacity = "1" } } - } } @@ -39,15 +38,18 @@ resource "aws_autoscaling_group" "k8s_servers_asg" { health_check_type = "EC2" force_delete = true - tag { - key = "provisioner" - value = "terraform" - propagate_at_launch = true + dynamic "tag" { + for_each = local.global_tags + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } } tag { - key = "environment" - value = var.environment + key = "Name" + value = "${var.common_prefix}-server-${var.environment}" propagate_at_launch = true } @@ -57,24 +59,6 @@ resource "aws_autoscaling_group" "k8s_servers_asg" { propagate_at_launch = true } - tag { - key = "uuid" - value = var.uuid - propagate_at_launch = true - } - - tag { - key = "scope" - value = "k8s-cluster" - propagate_at_launch = true - } - - tag { - key = "Name" - value = "k8s-server-${var.environment}" - propagate_at_launch = true - } - tag { key = "k8s.io/cluster-autoscaler/enabled" value = "" @@ -89,7 +73,7 @@ resource "aws_autoscaling_group" "k8s_servers_asg" { } resource "aws_autoscaling_group" "k8s_workers_asg" { - name = "k8s_workers" + name = "${var.common_prefix}-workers-asg-${var.environment}" vpc_zone_identifier = var.vpc_private_subnets lifecycle { @@ -127,15 +111,18 @@ resource "aws_autoscaling_group" "k8s_workers_asg" { health_check_type = "EC2" force_delete = true - tag { - key = "provisioner" - value = "terraform" - propagate_at_launch = true + dynamic "tag" { + for_each = local.global_tags + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } } tag { - key = "environment" - value = var.environment + key = "Name" + value = "${var.common_prefix}-worker-${var.environment}" propagate_at_launch = true } @@ -145,24 +132,6 @@ resource "aws_autoscaling_group" "k8s_workers_asg" { propagate_at_launch = true } - tag { - key = "uuid" - value = var.uuid - propagate_at_launch = true - } - - tag { - key = "scope" - value = "k8s-cluster" - propagate_at_launch = true - } - - tag { - key = "Name" - value = "k8s-worker-${var.environment}" - propagate_at_launch = true - } - tag { key = "k8s.io/cluster-autoscaler/enabled" value = "" diff --git a/bucket.tf b/bucket.tf deleted file mode 100644 index dea00c4..0000000 --- a/bucket.tf +++ /dev/null @@ -1,19 +0,0 @@ -resource "aws_s3_bucket" "k8s_cert_bucket" { - bucket = var.s3_bucket_name - - tags = merge( - local.tags, - { - Name = "k8s-s3-bucket-${var.environment}" - } - ) -} - -resource "aws_s3_bucket_public_access_block" "k8s_cert_bucket_access_block" { - bucket = aws_s3_bucket.k8s_cert_bucket.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} \ No newline at end of file diff --git a/data.tf b/data.tf index d05caa6..7b73cba 100644 --- a/data.tf +++ b/data.tf @@ -2,6 +2,10 @@ data "aws_iam_policy" "AmazonEC2ReadOnlyAccess" { arn = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" } +data "aws_iam_policy" "AmazonSSMManagedInstanceCore" { + arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + data "template_cloudinit_config" "k8s_server" { gzip = true base64_encode = true @@ -14,25 +18,40 @@ data "template_cloudinit_config" "k8s_server" { part { content_type = "text/x-shellscript" - content = templatefile("${path.module}/files/install_k8s_utils.sh", { k8s_version = var.k8s_version, install_longhorn = var.install_longhorn, }) + content = templatefile("${path.module}/files/install_k8s_utils.sh", { + k8s_version = var.k8s_version + }) } part { content_type = "text/x-shellscript" content = templatefile("${path.module}/files/install_k8s.sh", { - is_k8s_server = true, - k8s_version = var.k8s_version, - k8s_dns_domain = var.k8s_dns_domain, - k8s_pod_subnet = var.k8s_pod_subnet, - k8s_service_subnet = var.k8s_service_subnet, - s3_bucket_name = var.s3_bucket_name, - kube_api_port = var.kube_api_port, - control_plane_url = aws_lb.k8s-server-lb.dns_name, - install_longhorn = var.install_longhorn, - longhorn_release = var.longhorn_release, - install_nginx_ingress = var.install_nginx_ingress, - extlb_listener_http_port = var.extlb_listener_http_port, - extlb_listener_https_port = var.extlb_listener_https_port, + is_k8s_server = true, + k8s_version = var.k8s_version, + k8s_dns_domain = var.k8s_dns_domain, + k8s_pod_subnet = var.k8s_pod_subnet, + k8s_service_subnet = var.k8s_service_subnet, + kubeadm_ca_secret_name = local.kubeadm_ca_secret_name, + kubeadm_token_secret_name = local.kubeadm_token_secret_name, + kubeadm_cert_secret_name = local.kubeadm_cert_secret_name, + kubeconfig_secret_name = local.kubeconfig_secret_name, + kube_api_port = var.kube_api_port, + control_plane_url = aws_lb.k8s_server_lb.dns_name, + install_nginx_ingress = var.install_nginx_ingress, + nginx_ingress_release = var.nginx_ingress_release, + efs_persistent_storage = var.efs_persistent_storage, + efs_csi_driver_release = var.efs_csi_driver_release, + efs_filesystem_id = var.efs_persistent_storage ? aws_efs_file_system.k8s_persistent_storage[0].id : "", + install_certmanager = var.install_certmanager, + certmanager_release = var.certmanager_release, + install_node_termination_handler = var.install_node_termination_handler, + node_termination_handler_release = var.node_termination_handler_release, + certmanager_email_address = var.certmanager_email_address, + extlb_listener_http_port = var.extlb_listener_http_port, + extlb_listener_https_port = var.extlb_listener_https_port, + default_secret_placeholder = var.default_secret_placeholder, + expose_kubeapi = var.expose_kubeapi, + k8s_tls_san_public = local.k8s_tls_san_public }) } } @@ -49,16 +68,21 @@ data "template_cloudinit_config" "k8s_worker" { part { content_type = "text/x-shellscript" - content = templatefile("${path.module}/files/install_k8s_utils.sh", { k8s_version = var.k8s_version, install_longhorn = var.install_longhorn }) + content = templatefile("${path.module}/files/install_k8s_utils.sh", { + k8s_version = var.k8s_version + }) } part { content_type = "text/x-shellscript" content = templatefile("${path.module}/files/install_k8s_worker.sh", { - is_k8s_server = false, - s3_bucket_name = var.s3_bucket_name, - kube_api_port = var.kube_api_port, - control_plane_url = aws_lb.k8s-server-lb.dns_name, + is_k8s_server = false, + kubeadm_ca_secret_name = local.kubeadm_ca_secret_name, + kubeadm_token_secret_name = local.kubeadm_token_secret_name, + kubeadm_cert_secret_name = local.kubeadm_cert_secret_name, + kube_api_port = var.kube_api_port, + control_plane_url = aws_lb.k8s_server_lb.dns_name, + default_secret_placeholder = var.default_secret_placeholder, }) } } @@ -70,11 +94,7 @@ data "aws_instances" "k8s_servers" { ] instance_tags = { - k8s-instance-type = "k8s-server" - provisioner = "terraform" - environment = var.environment - uuid = var.uuid - scope = "k8s-cluster" + for tag, value in merge(local.global_tags, { k8s-instance-type = "k8s-server" }) : tag => value } instance_state_names = ["running"] @@ -87,11 +107,7 @@ data "aws_instances" "k8s_workers" { ] instance_tags = { - k8s-instance-type = "k8s-worker" - provisioner = "terraform" - environment = var.environment - uuid = var.uuid - scope = "k8s-cluster" + for tag, value in merge(local.global_tags, { k8s-instance-type = "k8s-worker" }) : tag => value } instance_state_names = ["running"] diff --git a/deployments/README.md b/deployments/README.md new file mode 100644 index 0000000..8ff46df --- /dev/null +++ b/deployments/README.md @@ -0,0 +1,141 @@ +## Deploy ECK on Kubernetes + +In this guide we will install [ECK](https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-deploy-eck.html) on Kubernetes. + +First of all install CRDs and ECK operator: + +``` +kubectl create -f https://download.elastic.co/downloads/eck/2.8.0/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/2.8.0/operator.yaml +``` + +Then we need to configure our Storage Class ([EFS CSI Driver with dynamic profisioning](https://github.com/kubernetes-sigs/aws-efs-csi-driver)) where elasticsearch will store and persist data: + +```yaml +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: efs-eck-sc # Don't change this name since is used on eck-elastic.yml deloyment +provisioner: efs.csi.aws.com +parameters: + provisioningMode: efs-ap + fileSystemId: # get the id form AWS console + directoryPerms: "755" + basePath: "/eck-storage-dynamic" # optional. Choose an appropriate name +``` + +Apply the above deployments and then we are ready to deploy elasticsearch: + +``` +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-elastic.yml +``` + +Check the status of the newly created pods, pv and pvc: + +``` +kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 0/1 Init:0/2 0 2s +k8s-eck-es-default-1 0/1 Init:0/2 0 2s +k8s-eck-es-default-2 0/1 Init:0/2 0 2s + +root@i-097c1a2b2f1022439:~/eck# kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-0d766371-f9a4-4210-abe5-077748808643 20Gi RWO Delete Bound default/elasticsearch-data-k8s-eck-es-default-0 efs-eck-sc 34s +pvc-6290aa54-f41b-4705-99fe-f69efddeb168 20Gi RWO Delete Bound default/elasticsearch-data-k8s-eck-es-default-1 efs-eck-sc 34s +pvc-e8e7a076-f8c3-4a93-8239-44b5ca8696fa 20Gi RWO Delete Bound default/elasticsearch-data-k8s-eck-es-default-2 efs-eck-sc 34s + +root@i-097c1a2b2f1022439:~/eck# kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +elasticsearch-data-k8s-eck-es-default-0 Bound pvc-0d766371-f9a4-4210-abe5-077748808643 20Gi RWO efs-eck-sc 35s +elasticsearch-data-k8s-eck-es-default-1 Bound pvc-6290aa54-f41b-4705-99fe-f69efddeb168 20Gi RWO efs-eck-sc 35s +elasticsearch-data-k8s-eck-es-default-2 Bound pvc-e8e7a076-f8c3-4a93-8239-44b5ca8696fa 20Gi RWO efs-eck-sc 35s +``` + +Wait until the elasticsearch pods are ready: + +``` +root@i-097c1a2b2f1022439:~/eck# kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 1/1 Running 0 3m3s +k8s-eck-es-default-1 1/1 Running 0 3m3s +k8s-eck-es-default-2 1/1 Running 0 3m3s +``` + +Now we can deploy Kibana with: + +``` +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-kibana.yml +``` + +Wait until kibana is up & running and check for the kibana service name: + +``` +root@i-097c1a2b2f1022439:~/eck# kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 1/1 Running 0 9m52s +k8s-eck-es-default-1 1/1 Running 0 9m52s +k8s-eck-es-default-2 1/1 Running 0 9m52s +k8s-eck-kibana-kb-56c4fb4bf8-vc9ct 1/1 Running 0 3m31s + +kubectl get svc +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +k8s-eck-es-default ClusterIP None 9200/TCP 9m54s +k8s-eck-es-http ClusterIP 10.107.103.161 9200/TCP 9m55s +k8s-eck-es-internal-http ClusterIP 10.101.251.215 9200/TCP 9m55s +k8s-eck-es-transport ClusterIP None 9300/TCP 9m55s +k8s-eck-kibana-kb-http ClusterIP 10.102.152.26 5601/TCP 3m34s +kubernetes ClusterIP 10.96.0.1 443/TCP 51m +``` + +Now create an ingress rule with the above deployment and apply it: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: eck-kibana-ingress + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" +spec: + ingressClassName: nginx + rules: + - host: eck.yourdomain.com # FQDN in a domain that you manage. Create a CNAME record that point to the public LB DNS name + http: + paths: + - pathType: Prefix + path: / + backend: + service: + name: k8s-eck-kibana-kb-http + port: + number: 5601 +``` + +Now apply filebeat and metricbeat deployments to get some data into elasticsearch: + +``` +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-filebeat.yml +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-metricbeat.yml +``` + +And wait that all the pods are ready: + +``` +root@i-097c1a2b2f1022439:~/eck# kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 1/1 Running 0 54m +k8s-eck-es-default-1 1/1 Running 0 54m +k8s-eck-es-default-2 1/1 Running 0 54m +k8s-eck-filebeat-beat-filebeat-76s9x 1/1 Running 4 (11m ago) 12m +k8s-eck-filebeat-beat-filebeat-pn77d 1/1 Running 4 (11m ago) 12m +k8s-eck-filebeat-beat-filebeat-wjkhm 1/1 Running 4 (11m ago) 12m +k8s-eck-kibana-kb-77d89694bc-vbp7s 1/1 Running 0 19m +k8s-eck-metricbeat-beat-metricbeat-8kpkl 1/1 Running 1 (7m36s ago) 8m1s +k8s-eck-metricbeat-beat-metricbeat-fl28t 1/1 Running 0 8m1s +k8s-eck-metricbeat-beat-metricbeat-knn2j 1/1 Running 1 (6m16s ago) 8m1s +``` + +Finally login to the Kibana UI on https://eck.yourdomain.com. Check [here](https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-deploy-kibana.html) how to get the elastic password. \ No newline at end of file diff --git a/deployments/eck-elastic.yml b/deployments/eck-elastic.yml new file mode 100644 index 0000000..b187f22 --- /dev/null +++ b/deployments/eck-elastic.yml @@ -0,0 +1,21 @@ +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: k8s-eck +spec: + version: 8.8.2 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false + volumeClaimTemplates: + - metadata: + name: elasticsearch-data # Do not change this name unless you set up a volume mount for the data path. + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: efs-eck-sc \ No newline at end of file diff --git a/deployments/eck-filebeat.yml b/deployments/eck-filebeat.yml new file mode 100644 index 0000000..148cbb8 --- /dev/null +++ b/deployments/eck-filebeat.yml @@ -0,0 +1,95 @@ +--- +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: k8s-eck-filebeat +spec: + type: filebeat + version: 8.8.2 + elasticsearchRef: + name: k8s-eck + kibanaRef: + name: k8s-eck-kibana + config: + filebeat.autodiscover.providers: + - node: ${NODE_NAME} + type: kubernetes + hints.default_config.enabled: "false" + templates: + - condition.equals.kubernetes.namespace: default + config: + - paths: ["/var/log/containers/*${data.kubernetes.container.id}.log"] + type: container + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: filebeat + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + containers: + - name: filebeat + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat + namespace: default +subjects: +- kind: ServiceAccount + name: filebeat + namespace: default +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deployments/eck-kibana.yml b/deployments/eck-kibana.yml new file mode 100644 index 0000000..6f90311 --- /dev/null +++ b/deployments/eck-kibana.yml @@ -0,0 +1,10 @@ +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: k8s-eck-kibana +spec: + version: 8.8.2 + count: 1 + elasticsearchRef: + name: k8s-eck \ No newline at end of file diff --git a/deployments/eck-metricbeat.yml b/deployments/eck-metricbeat.yml new file mode 100644 index 0000000..0249643 --- /dev/null +++ b/deployments/eck-metricbeat.yml @@ -0,0 +1,173 @@ +--- +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: k8s-eck-metricbeat +spec: + type: metricbeat + version: 8.8.2 + elasticsearchRef: + name: k8s-eck + kibanaRef: + name: k8s-eck-kibana + config: + metricbeat: + autodiscover: + providers: + - hints: + default_config: {} + enabled: "true" + node: ${NODE_NAME} + type: kubernetes + modules: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + process: + include_top_n: + by_cpu: 5 + by_memory: 5 + processes: + - .* + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event: + when: + regexp: + system: + filesystem: + mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/) + - module: kubernetes + period: 10s + node: ${NODE_NAME} + hosts: + - https://${NODE_NAME}:10250 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl: + verification_mode: none + metricsets: + - node + - system + - pod + - container + - volume + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: metricbeat + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + containers: + - args: + - -e + - -c + - /etc/beat.yml + - -system.hostfs=/hostfs + name: metricbeat + volumeMounts: + - mountPath: /hostfs/sys/fs/cgroup + name: cgroup + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostfs/proc + name: proc + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /proc + name: proc +--- +# permissions needed for metricbeat +# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + namespace: default +rules: +- apiGroups: + - "" + resources: + - nodes + - namespaces + - events + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "extensions" + resources: + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - deployments + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat + namespace: default +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: default +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/efs.tf b/efs.tf new file mode 100644 index 0000000..bc4a045 --- /dev/null +++ b/efs.tf @@ -0,0 +1,19 @@ +resource "aws_efs_file_system" "k8s_persistent_storage" { + count = var.efs_persistent_storage ? 1 : 0 + creation_token = "${var.common_prefix}-efs-persistent-storage-${var.environment}" + encrypted = true + + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-efs-persistent-storage-${var.environment}") + } + ) +} + +resource "aws_efs_mount_target" "k8s_persistent_storage_mount_target" { + count = var.efs_persistent_storage ? length(var.vpc_private_subnets) : 0 + file_system_id = aws_efs_file_system.k8s_persistent_storage[0].id + subnet_id = var.vpc_private_subnets[count.index] + security_groups = [aws_security_group.efs_sg[0].id] +} \ No newline at end of file diff --git a/examples/.terraform.lock.hcl b/examples/.terraform.lock.hcl index 56ca886..5b4cf11 100644 --- a/examples/.terraform.lock.hcl +++ b/examples/.terraform.lock.hcl @@ -2,21 +2,24 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "4.9.0" + version = "5.9.0" hashes = [ - "h1:GtmIOZMkKmr9tMLWouHWiGXmKEL/diOTNar5XfOVLjs=", - "zh:084b83aef3335ad4f5e4b8323c6fe43c1ff55e17a7647c6a5cad6af519f72b42", - "zh:132e47ce69f14de4523b84b213cedf7173398acda14245b1ffe7747aac50f050", - "zh:2068baef7dfce3613f3b4f27314175e971f8db68d9cde9ec30b5659f80c68c6c", - "zh:63c6f489683d5f1ac55e82a0df387143ed22701d5f22c109a4d5c9924dd4e437", - "zh:8115fd21965954fa4568c09331e05bb29da967fab8d077419aed09954378e216", - "zh:8efdc95fde108f777ed9c79ae25dc17aea9771903250f5c5c8a4c726b90a345f", + "h1:mvg6WWqqUvgUq6wYCWg/zqpND/5yIz3plIL1IOR50Rs=", + "zh:032424d4686ce2ff7c5a4a738491635616afbf6e06b3e7e6a754baa031d1265d", + "zh:1e530b4020544ec94e1fe7b1e4296640eb12cf1bf4f79cd6429ff2c4e6fffaf3", + "zh:24d2eee57a4c78039959dd9bb6dff2b75ed0483d44929550c067c3488307dc62", + "zh:3ad6d736722059664e790a358eacf0e0e60973ec44e70142fb503275de2116c1", + "zh:3f34d81acf86c61ddd271e9c4b8215765037463c3fe3c7aea1dc32a509020cfb", + "zh:65a04aa615fc320059a0871702c83b6be10bce2064056096b46faffe768a698e", + "zh:7fb56c3ce1fe77983627e2931e7c7b73152180c4dfb03e793413d0137c85d6b2", + "zh:90c94cb9d7352468bcd5ba21a56099fe087a072b1936d86f47d54c2a012b708a", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9d42a7bc34d84b70c1d1bcc215cabd63abbcbd0352b70bd84da6c3916634932f", - "zh:aacbcceb241aa475888c0869e87593182edeced3170c76a0c960dd9c905df449", - "zh:c7fe7904511052e4102870256819a1917177572cf684f0611ebf767f9c1fbaa8", - "zh:c8e07c3424663d1d0e7e32f4ade8099c19f6326d37c6da98104d90c986ff66fc", - "zh:e47cafbd38b56ef14fd8d727b4ffea847c166b1c684f585ee5fb78983b537248", + "zh:a109c5f01ed48852fe17847fa8a116dfdb81500794a9cf7e5ef92ea6dec20431", + "zh:a27c5396077a36ac2801d4c1c1132201a9225a65bba0e3b3aded9cc18f2c38ff", + "zh:a86ad796ccb0f2cb8f0ca069c774dbf74964edd3282529726816c72e22164b3c", + "zh:bda8afc64091a2a72e0cc38fde937b2163b1b072a5c41310d255901207571afd", + "zh:d22473894cd7e94b7a971793dd07309569f82913a10e4bd6c22e04f362f03bb9", + "zh:f4dbb6d13511290a5274f5b202e6d9997643f86e4c48e8c5e3c204121082851a", ] } diff --git a/examples/main.tf b/examples/main.tf index 1f64281..f64bc95 100644 --- a/examples/main.tf +++ b/examples/main.tf @@ -11,22 +11,60 @@ variable "environment" { } variable "AWS_REGION" { - default = "" + default = "" +} + +variable "my_public_ip_cidr" { + default = "" +} + +variable "vpc_cidr_block" { + default = "" +} + +variable "certmanager_email_address" { + default = "" +} + +variable "ssk_key_pair_name" { + default = "" +} + +module "private-vpc" { + region = var.AWS_REGION + my_public_ip_cidr = var.my_public_ip_cidr + vpc_cidr_block = var.vpc_cidr_block + environment = var.environment + source = "github.com/garutilorenzo/aws-terraform-examples/private-vpc" +} + +output "private_subnets_ids" { + value = module.private-vpc.private_subnet_ids +} + +output "public_subnets_ids" { + value = module.private-vpc.public_subnet_ids +} + +output "vpc_id" { + value = module.private-vpc.vpc_id } module "k8s-cluster" { - ssk_key_pair_name = "" - uuid = "" - environment = var.environment - vpc_id = "" - vpc_private_subnets = "" - vpc_public_subnets = "" - vpc_subnet_cidr = "" - PATH_TO_PUBLIC_LB_CERT = "" - PATH_TO_PUBLIC_LB_KEY = "" - install_nginx_ingress = true - install_longhorn = true - source = "github.com/garutilorenzo/k8s-aws-terraform-cluster" + ssk_key_pair_name = var.ssk_key_pair_name + environment = var.environment + vpc_id = module.private-vpc.vpc_id + vpc_private_subnets = module.private-vpc.private_subnet_ids + vpc_public_subnets = module.private-vpc.public_subnet_ids + vpc_subnet_cidr = var.vpc_cidr_block + my_public_ip_cidr = var.my_public_ip_cidr + create_extlb = true + install_nginx_ingress = true + efs_persistent_storage = true + expose_kubeapi = true + install_certmanager = true + certmanager_email_address = var.certmanager_email_address + source = "github.com/garutilorenzo/k8s-aws-terraform-cluster" } output "k8s_dns_name" { diff --git a/files/install_k8s.sh b/files/install_k8s.sh index c75cb3f..ec53496 100644 --- a/files/install_k8s.sh +++ b/files/install_k8s.sh @@ -3,7 +3,7 @@ render_kubeinit(){ HOSTNAME=$(hostname) -ADVERTISE_ADDR=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p') +ADVERTISE_ADDR=$(ip -o route get to 8.8.8.8 | grep -Po '(?<=src )(\S+)') cat <<-EOF > /root/kubeadm-init-config.yaml --- @@ -19,7 +19,7 @@ bootstrapTokens: kind: InitConfiguration localAPIEndpoint: advertiseAddress: $ADVERTISE_ADDR - bindPort: ${kube_api_port } + bindPort: ${kube_api_port} nodeRegistration: criSocket: /run/containerd/containerd.sock imagePullPolicy: IfNotPresent @@ -28,15 +28,21 @@ nodeRegistration: --- apiServer: timeoutForControlPlane: 4m0s + certSANs: + - $HOSTNAME + - $ADVERTISE_ADDR + %{~ if expose_kubeapi ~} + - ${k8s_tls_san_public} + %{~ endif ~} apiVersion: kubeadm.k8s.io/v1beta3 certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controllerManager: {} dns: {} -imageRepository: k8s.gcr.io +imageRepository: registry.k8s.io kind: ClusterConfiguration kubernetesVersion: ${k8s_version} -controlPlaneEndpoint: ${control_plane_url}:${kube_api_port } +controlPlaneEndpoint: ${control_plane_url}:${kube_api_port} networking: dnsDomain: ${k8s_dns_domain} podSubnet: ${k8s_pod_subnet} @@ -52,11 +58,41 @@ cgroupDriver: systemd EOF } -wait_for_s3_object(){ - until aws s3 ls s3://${s3_bucket_name}/ca.txt +wait_lb() { +while [ true ] +do + curl --output /dev/null --silent -k https://${control_plane_url}:${kube_api_port} + if [[ "$?" -eq 0 ]]; then + break + fi + sleep 5 + echo "wait for LB" +done +} + +wait_for_ca_secret(){ + res_ca=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) + while [[ -z "$res_ca" || "$res_ca" == "${default_secret_placeholder}" ]] + do + echo "Waiting the ca hash ..." + res_ca=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) + sleep 1 + done + + res_cert=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_cert_secret_name} | jq -r .SecretString) + while [[ -z "$res_cert" || "$res_cert" == "${default_secret_placeholder}" ]] do - echo "Waiting the ca hash ..." - sleep 10 + echo "Waiting the ca hash ..." + res_cert=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_cert_secret_name} | jq -r .SecretString) + sleep 1 + done + + res_token=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_token_secret_name} | jq -r .SecretString) + while [[ -z "$res_token" || "$res_token" == "${default_secret_placeholder}" ]] + do + echo "Waiting the ca hash ..." + res_token=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_token_secret_name} | jq -r .SecretString) + sleep 1 done } @@ -68,7 +104,7 @@ wait_for_pods(){ } wait_for_masters(){ - until kubectl get nodes -o wide | grep 'control-plane,master'; do + until kubectl get nodes -o wide | grep 'control-plane'; do echo 'Waiting for k8s control-planes' sleep 5 done @@ -86,10 +122,10 @@ setup_env(){ render_kubejoin(){ HOSTNAME=$(hostname) -ADVERTISE_ADDR=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p') -CA_HASH=$(aws s3 cp s3://${s3_bucket_name}/ca.txt -) -KUBEADM_CERT=$(aws s3 cp s3://${s3_bucket_name}/kubeadm_cert.txt -) -KUBEADM_TOKEN=$(aws s3 cp s3://${s3_bucket_name}/kubeadm_token.txt -) +ADVERTISE_ADDR=$(ip -o route get to 8.8.8.8 | grep -Po '(?<=src )(\S+)') +CA_HASH=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) +KUBEADM_CERT=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_cert_secret_name} | jq -r .SecretString) +KUBEADM_TOKEN=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_token_secret_name} | jq -r .SecretString) cat <<-EOF > /root/kubeadm-join-master.yaml --- @@ -119,17 +155,11 @@ EOF } render_nginx_config(){ -cat <<-EOF > /root/nginx-ingress-config.yaml +cat << 'EOF' > "$NGINX_RESOURCES_FILE" --- apiVersion: v1 kind: Service metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 name: ingress-nginx-controller namespace: ingress-nginx spec: @@ -155,33 +185,129 @@ spec: apiVersion: v1 data: allow-snippet-annotations: "true" - use-forwarded-headers: "true" - compute-full-forwarded-for: "true" enable-real-ip: "true" - forwarded-for-header: "X-Forwarded-For" proxy-real-ip-cidr: "0.0.0.0/0" + proxy-body-size: "20m" + use-proxy-protocol: "true" kind: ConfigMap metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.1 - helm.sh/chart: ingress-nginx-4.0.16 + app.kubernetes.io/version: ${nginx_ingress_release} name: ingress-nginx-controller namespace: ingress-nginx EOF } +install_and_configure_nginx(){ + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-${nginx_ingress_release}/deploy/static/provider/baremetal/deploy.yaml + NGINX_RESOURCES_FILE=/root/nginx-ingress-resources.yaml + render_nginx_config + kubectl apply -f $NGINX_RESOURCES_FILE +} + +render_staging_issuer(){ +STAGING_ISSUER_RESOURCE=$1 +cat << 'EOF' > "$STAGING_ISSUER_RESOURCE" +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging + namespace: cert-manager +spec: + acme: + # The ACME server URL + server: https://acme-staging-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: ${certmanager_email_address} + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-staging + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: nginx +EOF +} + +render_prod_issuer(){ +PROD_ISSUER_RESOURCE=$1 +cat << 'EOF' > "$PROD_ISSUER_RESOURCE" +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod + namespace: cert-manager +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: ${certmanager_email_address} + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-prod + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: nginx +EOF +} + +install_and_configure_certmanager(){ + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${certmanager_release}/cert-manager.yaml + render_staging_issuer /root/staging_issuer.yaml + render_prod_issuer /root/prod_issuer.yaml + + # Wait cert-manager to be ready + until kubectl get pods -n cert-manager | grep 'Running'; do + echo 'Waiting for cert-manager to be ready' + sleep 15 + done + + kubectl create -f /root/prod_issuer.yaml + kubectl create -f /root/staging_issuer.yaml +} + +install_and_configure_csi_driver(){ + git clone https://github.com/kubernetes-sigs/aws-efs-csi-driver.git + cd aws-efs-csi-driver/ + git checkout tags/${efs_csi_driver_release} -b kube_deploy_${efs_csi_driver_release} + kubectl apply -k deploy/kubernetes/overlays/stable/ + + # Uncomment this to mount the EFS share on the first k8s-server node + # mkdir /efs + # aws_region="$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)" + # mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport ${efs_filesystem_id}.efs.$aws_region.amazonaws.com:/ /efs +} + k8s_join(){ kubeadm join --config /root/kubeadm-join-master.yaml mkdir ~/.kube cp /etc/kubernetes/admin.conf ~/.kube/config + + # Upload kubeconfig on AWS secret manager + cat ~/.kube/config | sed 's/server: https:\/\/127.0.0.1:6443/server: https:\/\/${control_plane_url}:${kube_api_port}/' > /root/kube.conf + aws secretsmanager update-secret --secret-id ${kubeconfig_secret_name} --secret-string file:///root/kube.conf +} + +wait_for_secretsmanager(){ + res=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) + while [[ -z "$res" ]] + do + echo "Waiting the ca hash ..." + res=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) + sleep 1 + done } -generate_s3_object(){ +generate_secrets(){ + wait_for_secretsmanager HASH=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //') echo $HASH > /tmp/ca.txt @@ -191,9 +317,9 @@ generate_s3_object(){ CERT=$(kubeadm init phase upload-certs --upload-certs | tail -n 1) echo $CERT > /tmp/kubeadm_cert.txt - aws s3 cp /tmp/ca.txt s3://${s3_bucket_name}/ca.txt - aws s3 cp /tmp/kubeadm_cert.txt s3://${s3_bucket_name}/kubeadm_cert.txt - aws s3 cp /tmp/kubeadm_token.txt s3://${s3_bucket_name}/kubeadm_token.txt + aws secretsmanager update-secret --secret-id ${kubeadm_ca_secret_name} --secret-string file:///tmp/ca.txt + aws secretsmanager update-secret --secret-id ${kubeadm_cert_secret_name} --secret-string file:///tmp/kubeadm_cert.txt + aws secretsmanager update-secret --secret-id ${kubeadm_token_secret_name} --secret-string file:///tmp/kubeadm_token.txt } k8s_init(){ @@ -203,34 +329,41 @@ k8s_init(){ } setup_cni(){ - kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml + until kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml; do + echo "Trying to install CNI flannel" + done } first_instance=$(aws ec2 describe-instances --filters Name=tag-value,Values=k8s-server Name=instance-state-name,Values=running --query 'sort_by(Reservations[].Instances[], &LaunchTime)[:-1].[InstanceId]' --output text | head -n1) instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) -control_plane_status=$(curl -o /dev/null -L -k -s -w '%%{http_code}' https://${control_plane_url}:${kube_api_port}) -if [[ "$first_instance" == "$instance_id" ]] && [[ "$control_plane_status" -ne 403 ]]; then +if [[ "$first_instance" == "$instance_id" ]]; then render_kubeinit k8s_init setup_env wait_for_pods setup_cni - generate_s3_object + generate_secrets echo "Wait 180 seconds for control-planes to join" sleep 180 wait_for_masters %{ if install_nginx_ingress } - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/baremetal/deploy.yaml - render_nginx_config - kubectl apply -f /root/nginx-ingress-config.yaml + install_and_configure_nginx + %{ endif } + %{ if install_certmanager } + install_and_configure_certmanager + %{ endif } + %{ if efs_persistent_storage } + install_and_configure_csi_driver %{ endif } - %{ if install_longhorn } - kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/${longhorn_release}/deploy/longhorn.yaml - kubectl create -f https://raw.githubusercontent.com/longhorn/longhorn/${longhorn_release}/examples/storageclass.yaml + %{ if install_node_termination_handler } + #Install node termination handler + echo 'Install node termination handler' + kubectl apply -f https://github.com/aws/aws-node-termination-handler/releases/download/${node_termination_handler_release}/all-resources.yaml %{ endif } else - wait_for_s3_object + wait_for_ca_secret render_kubejoin + wait_lb k8s_join fi \ No newline at end of file diff --git a/files/install_k8s_utils.sh b/files/install_k8s_utils.sh index 06d4f32..7fecf07 100644 --- a/files/install_k8s_utils.sh +++ b/files/install_k8s_utils.sh @@ -1,7 +1,27 @@ #!/bin/bash -DEBIAN_FRONTEND=noninteractive -export DEBIAN_FRONTEND +check_os() { + name=$(cat /etc/os-release | grep ^NAME= | sed 's/"//g') + clean_name=$${name#*=} + + version=$(cat /etc/os-release | grep ^VERSION_ID= | sed 's/"//g') + clean_version=$${version#*=} + major=$${clean_version%.*} + minor=$${clean_version#*.} + + if [[ "$clean_name" == "Ubuntu" ]]; then + operating_system="ubuntu" + elif [[ "$clean_name" == "Amazon Linux" ]]; then + operating_system="amazonlinux" + else + operating_system="undef" + fi + + echo "K3S install process running on: " + echo "OS: $operating_system" + echo "OS Major Release: $major" + echo "OS Minor Release: $minor" +} render_config(){ cat <<-EOF | tee /etc/modules-load.d/containerd.conf @@ -22,71 +42,186 @@ sudo sysctl --system } preflight(){ - apt-get update && apt-get upgrade -y - - apt-get install -y \ - ca-certificates \ - unzip \ - software-properties-common \ - curl \ - gnupg \ - openssl \ - lsb-release \ - apt-transport-https - - render_config + apt-get update && apt-get upgrade -y + + apt-get install -y \ + ca-certificates \ + unzip \ + software-properties-common \ + curl \ + gnupg \ + openssl \ + lsb-release \ + apt-transport-https \ + jq + + render_config +} + +render_containerd_service(){ +cat <<-EOF > /etc/systemd/system/containerd.service +# Copyright The containerd Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target +[Service] +#uncomment to enable the experimental sbservice (sandboxed) version of containerd/cri integration +#Environment="ENABLE_CRI_SANDBOXES=sandboxed" +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=infinity +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 +[Install] +WantedBy=multi-user.target +EOF +} + +preflight_amz(){ + yum check-update + yum upgrade -y + yum install -y curl unzip jq openssl + + render_config +} + +setup_repos_amz(){ +cat < /dev/null + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list - apt-get update + apt-get update +} + +render_crictl_conf(){ +cat <<-EOF | tee /etc/crictl.yaml +--- +runtime-endpoint: unix:///var/run/containerd/containerd.sock +image-endpoint: unix:///var/run/containerd/containerd.sock +EOF } setup_cri(){ - apt-get install -y containerd.io - mkdir -p /etc/containerd - cat /etc/containerd/config.toml | grep -Fx "[grpc]" - res=$? - if [ $res -ne 0 ]; then - containerd config default | tee /etc/containerd/config.toml - fi - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml - systemctl restart containerd - systemctl enable containerd + apt-get install -y containerd.io + mkdir -p /etc/containerd + cat /etc/containerd/config.toml | grep -Fx "[grpc]" + res=$? + if [ $res -ne 0 ]; then + containerd config default | tee /etc/containerd/config.toml + fi + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + + render_crictl_conf + + systemctl restart containerd + systemctl enable containerd } install_k8s_utils(){ - apt-get install -y kubelet=${k8s_version}* kubeadm=${k8s_version}* kubectl=${k8s_version}* - apt-mark hold kubelet kubeadm kubectl + apt-get install -y kubelet=${k8s_version}* kubeadm=${k8s_version}* kubectl=${k8s_version}* + apt-mark hold kubelet kubeadm kubectl } -preflight -install_aws_cli -setup_repos -setup_cri -install_k8s_utils -%{ if install_longhorn } -preflight_longhorn -%{ endif } \ No newline at end of file +check_os + +if [[ "$operating_system" == "ubuntu" ]]; then + DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND + + preflight + install_aws_cli + setup_repos + setup_cri + install_k8s_utils +fi + +if [[ "$operating_system" == "amazonlinux" ]]; then + preflight_amz + setup_repos_amz + setup_cri_amz + install_k8s_utils_amz +fi \ No newline at end of file diff --git a/files/install_k8s_worker.sh b/files/install_k8s_worker.sh index 5039b29..d3d6db1 100644 --- a/files/install_k8s_worker.sh +++ b/files/install_k8s_worker.sh @@ -1,19 +1,41 @@ #!/bin/bash -wait_for_s3_object(){ - until aws s3 ls s3://${s3_bucket_name}/ca.txt +wait_lb() { +while [ true ] +do + curl --output /dev/null --silent -k https://${control_plane_url}:${kube_api_port} + if [[ "$?" -eq 0 ]]; then + break + fi + sleep 5 + echo "wait for LB" +done +} + +wait_for_ca_secret(){ + res=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) + while [[ -z "$res" || "$res" == "${default_secret_placeholder}" ]] + do + echo "Waiting the ca hash ..." + res=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) + sleep 1 + done + + res_token=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_token_secret_name} | jq -r .SecretString) + while [[ -z "$res_token" || "$res_token" == "${default_secret_placeholder}" ]] do echo "Waiting the ca hash ..." - sleep 10 + res_token=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_token_secret_name} | jq -r .SecretString) + sleep 1 done } render_kubejoin(){ HOSTNAME=$(hostname) -ADVERTISE_ADDR=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p') -CA_HASH=$(aws s3 cp s3://${s3_bucket_name}/ca.txt -) -KUBEADM_TOKEN=$(aws s3 cp s3://${s3_bucket_name}/kubeadm_token.txt -) +ADVERTISE_ADDR=$(ip -o route get to 8.8.8.8 | grep -Po '(?<=src )(\S+)') +CA_HASH=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_ca_secret_name} | jq -r .SecretString) +KUBEADM_TOKEN=$(aws secretsmanager get-secret-value --secret-id ${kubeadm_token_secret_name} | jq -r .SecretString) cat <<-EOF > /root/kubeadm-join-worker.yaml --- @@ -44,11 +66,8 @@ k8s_join(){ kubeadm join --config /root/kubeadm-join-worker.yaml } -until $(curl -k --output /dev/null --silent --head -X GET https://${control_plane_url}:${kube_api_port}); do - printf '.' - sleep 5 -done +wait_lb -wait_for_s3_object +wait_for_ca_secret render_kubejoin k8s_join diff --git a/iam.tf b/iam.tf index 7bd6e72..f800b49 100644 --- a/iam.tf +++ b/iam.tf @@ -1,18 +1,17 @@ resource "aws_iam_instance_profile" "k8s_instance_profile" { - name = var.instance_profile_name + name = "${var.common_prefix}-ec2-instance-profile--${var.environment}" role = aws_iam_role.k8s_iam_role.name tags = merge( - local.tags, + local.global_tags, { - Name = "k8s-instance-pofile-${var.environment}" + "Name" = lower("${var.common_prefix}-ec2-instance-profile--${var.environment}") } ) - } resource "aws_iam_role" "k8s_iam_role" { - name = var.iam_role_name + name = "${var.common_prefix}-iam-role-${var.environment}" assume_role_policy = jsonencode({ Version = "2012-10-17" @@ -29,18 +28,72 @@ resource "aws_iam_role" "k8s_iam_role" { }) tags = merge( - local.tags, + local.global_tags, { - Name = "k8s-iam-role-${var.environment}" + "Name" = lower("${var.common_prefix}-iam-role-${var.environment}") } ) +} + +resource "aws_iam_policy" "cluster_autoscaler" { + name = "${var.common_prefix}-cluster-autoscaler-policy-${var.environment}" + path = "/" + description = "Cluster autoscaler policy" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "ec2:DescribeLaunchTemplateVersions" + ], + Resource = [ + "${aws_launch_template.k8s_server.arn}", + "${aws_launch_template.k8s_worker.arn}" + ], + Condition = { + StringEquals = { + for tag, value in local.global_tags : "aws:ResourceTag/${tag}" => value + } + } + }, + { + Effect = "Allow" + Action = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:DescribeTags", + ], + Resource = [ + "${aws_autoscaling_group.k8s_servers_asg.arn}", + "${aws_autoscaling_group.k8s_workers_asg.arn}" + ], + Condition = { + StringEquals = { + for tag, value in local.global_tags : "aws:ResourceTag/${tag}" => value + } + } + }, + ] + }) + + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-cluster-autoscaler-policy-${var.environment}") + } + ) } -resource "aws_iam_policy" "s3_bucket_policy" { - name = "S3BucketPolicy" +resource "aws_iam_policy" "aws_efs_csi_driver_policy" { + count = var.efs_persistent_storage ? 1 : 0 + name = "${var.common_prefix}-csi-driver-policy-${var.environment}" path = "/" - description = "S3 Bucket Policy" + description = "AWS EFS CSI driver policy" policy = jsonencode({ Version = "2012-10-17" @@ -48,72 +101,141 @@ resource "aws_iam_policy" "s3_bucket_policy" { { Effect = "Allow" Action = [ - "s3:ListBucket" + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones" ], Resource = [ - "${aws_s3_bucket.k8s_cert_bucket.arn}" + "*" ] }, { Effect = "Allow" Action = [ - "s3:PutObject", - "s3:GetObject", - "s3:DeleteObject" + "elasticfilesystem:CreateAccessPoint" ], Resource = [ - "${aws_s3_bucket.k8s_cert_bucket.arn}/*" - ] + "*" + ], + Condition = { + StringLike = { + "aws:RequestTag/efs.csi.aws.com/cluster" = "true" + } + } + }, + { + Effect = "Allow" + Action = [ + "elasticfilesystem:TagResource" + ], + Resource = [ + "*" + ], + Condition = { + StringLike = { + "aws:ResourceTag/efs.csi.aws.com/cluster" = "true" + } + } + }, + { + Effect = "Allow" + Action = [ + "elasticfilesystem:DeleteAccessPoint" + ], + Resource = [ + "*" + ], + Condition = { + StringEquals = { + "aws:ResourceTag/efs.csi.aws.com/cluster" = "true" + } + } }, ] }) tags = merge( - local.tags, + local.global_tags, { - Name = "k8s-s3-bucket-policy-${var.environment}" + "Name" = lower("${var.common_prefix}-csi-driver-policy-${var.environment}") } ) } -# resource "aws_iam_policy" "cluster_autoscaler" { -# name = "ClusterAutoscalerPolicy" -# path = "/" -# description = "Cluster autoscaler policy" - -# policy = jsonencode({ -# Version = "2012-10-17" -# Statement = [ -# { -# Effect = "Allow" -# Action = [ -# "autoscaling:DescribeAutoScalingGroups", -# "autoscaling:DescribeAutoScalingInstances", -# "autoscaling:DescribeLaunchConfigurations", -# "autoscaling:SetDesiredCapacity", -# "autoscaling:TerminateInstanceInAutoScalingGroup", -# "autoscaling:DescribeTags", -# "ec2:DescribeLaunchTemplateVersions" -# ], -# Resource = [ -# "*" -# ] -# } -# ] -# }) - -# tags = { -# environment = "${var.environment}" -# provisioner = "terraform" -# } -# } +resource "aws_iam_policy" "allow_secrets_manager" { + name = "${var.common_prefix}-secrets-manager-policy-${var.environment}" + path = "/" + description = "Secrets Manager Policy" -resource "aws_iam_role_policy_attachment" "attach_ec2_ro_policy" { + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "secretsmanager:GetSecretValue", + "secretsmanager:UpdateSecret", + "secretsmanager:DeleteSecret", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecrets", + "secretsmanager:CreateSecret", + "secretsmanager:PutSecretValue" + ], + Resource = [ + "${aws_secretsmanager_secret.kubeconfig_secret.arn}", + "${aws_secretsmanager_secret.kubeadm_ca.arn}", + "${aws_secretsmanager_secret.kubeadm_token.arn}", + "${aws_secretsmanager_secret.kubeadm_cert.arn}" + ], + Condition = { + StringEquals = { + for tag, value in local.global_tags : "aws:ResourceTag/${tag}" => value + } + } + }, + { + Effect = "Allow" + Action = [ + "secretsmanager:ListSecrets" + ], + Resource = [ + "*" + ], + } + ] + }) + + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-secrets-manager-policy-${var.environment}") + } + ) +} + +resource "aws_iam_role_policy_attachment" "attach_ssm_policy" { role = aws_iam_role.k8s_iam_role.name - policy_arn = data.aws_iam_policy.AmazonEC2ReadOnlyAccess.arn + policy_arn = data.aws_iam_policy.AmazonSSMManagedInstanceCore.arn } -resource "aws_iam_role_policy_attachment" "attach_s3_bucket_policy" { +resource "aws_iam_role_policy_attachment" "attach_cluster_autoscaler_policy" { role = aws_iam_role.k8s_iam_role.name - policy_arn = aws_iam_policy.s3_bucket_policy.arn + policy_arn = aws_iam_policy.cluster_autoscaler.arn +} + +resource "aws_iam_role_policy_attachment" "attach_aws_efs_csi_driver_policy" { + count = var.efs_persistent_storage ? 1 : 0 + role = aws_iam_role.k8s_iam_role.name + policy_arn = aws_iam_policy.aws_efs_csi_driver_policy[0].arn +} + +resource "aws_iam_role_policy_attachment" "attach_allow_secrets_manager_policy" { + role = aws_iam_role.k8s_iam_role.name + policy_arn = aws_iam_policy.allow_secrets_manager.arn +} + +resource "aws_iam_role_policy_attachment" "attach_ec2_ro_policy" { + role = aws_iam_role.k8s_iam_role.name + policy_arn = data.aws_iam_policy.AmazonEC2ReadOnlyAccess.arn } \ No newline at end of file diff --git a/k8slb.tf b/k8slb.tf index 4ff73f7..96de489 100644 --- a/k8slb.tf +++ b/k8slb.tf @@ -1,5 +1,5 @@ -resource "aws_lb" "k8s-server-lb" { - name = var.k8s_internal_lb_name +resource "aws_lb" "k8s_server_lb" { + name = "${var.common_prefix}-int-lb-${var.environment}" load_balancer_type = "network" internal = "true" subnets = var.vpc_private_subnets @@ -7,40 +7,40 @@ resource "aws_lb" "k8s-server-lb" { enable_cross_zone_load_balancing = true tags = merge( - local.tags, + local.global_tags, { - Name = "lb-${var.k8s_internal_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-int-lb-${var.environment}") } ) } -resource "aws_lb_listener" "k8s-server-listener" { - load_balancer_arn = aws_lb.k8s-server-lb.arn +resource "aws_lb_listener" "k8s_server_listener" { + load_balancer_arn = aws_lb.k8s_server_lb.arn protocol = "TCP" port = var.kube_api_port default_action { type = "forward" - target_group_arn = aws_lb_target_group.k8s-server-tg.arn + target_group_arn = aws_lb_target_group.k8s_server_tg.arn } tags = merge( - local.tags, + local.global_tags, { - Name = "lb-listener-${var.k8s_internal_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-kubeapi-listener-${var.environment}") } ) } -resource "aws_lb_target_group" "k8s-server-tg" { +resource "aws_lb_target_group" "k8s_server_tg" { port = var.kube_api_port protocol = "TCP" vpc_id = var.vpc_id preserve_client_ip = false depends_on = [ - aws_lb.k8s-server-lb + aws_lb.k8s_server_lb ] health_check { @@ -51,15 +51,22 @@ resource "aws_lb_target_group" "k8s-server-tg" { lifecycle { create_before_destroy = true } + + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-internal-lb-tg-kubeapi-${var.environment}") + } + ) } -resource "aws_autoscaling_attachment" "target" { +resource "aws_autoscaling_attachment" "k8s_server_target_kubeapi" { depends_on = [ aws_autoscaling_group.k8s_servers_asg, - aws_lb_target_group.k8s-server-tg + aws_lb_target_group.k8s_server_tg ] autoscaling_group_name = aws_autoscaling_group.k8s_servers_asg.name - lb_target_group_arn = aws_lb_target_group.k8s-server-tg.arn + lb_target_group_arn = aws_lb_target_group.k8s_server_tg.arn } \ No newline at end of file diff --git a/lb.tf b/lb.tf index c0cfd40..54561c6 100644 --- a/lb.tf +++ b/lb.tf @@ -1,57 +1,54 @@ -resource "aws_lb" "external-lb" { - count = var.install_nginx_ingress ? 1 : 0 - name = var.k8s_ext_lb_name - load_balancer_type = "application" - security_groups = [aws_security_group.k8s-public-lb[count.index].id] +resource "aws_lb" "external_lb" { + count = var.create_extlb ? 1 : 0 + name = "${var.common_prefix}-ext-lb-${var.environment}" + load_balancer_type = "network" internal = "false" subnets = var.vpc_public_subnets enable_cross_zone_load_balancing = true tags = merge( - local.tags, + local.global_tags, { - Name = "${var.k8s_ext_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-ext-lb-${var.environment}") } ) - } # HTTP -resource "aws_lb_listener" "external-lb-listener-http" { - count = var.install_nginx_ingress ? 1 : 0 - load_balancer_arn = aws_lb.external-lb[count.index].arn +resource "aws_lb_listener" "external_lb_listener_http" { + count = var.create_extlb ? 1 : 0 + load_balancer_arn = aws_lb.external_lb[count.index].arn - protocol = "HTTP" + protocol = "TCP" port = var.extlb_http_port default_action { type = "forward" - target_group_arn = aws_lb_target_group.external-lb-tg-http[count.index].arn + target_group_arn = aws_lb_target_group.external_lb_tg_http[count.index].arn } tags = merge( - local.tags, + local.global_tags, { - Name = "lb-http-listener-${var.k8s_ext_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-http-listener-${var.environment}") } ) } -resource "aws_lb_target_group" "external-lb-tg-http" { - count = var.install_nginx_ingress ? 1 : 0 - port = var.extlb_listener_http_port - protocol = "HTTP" - vpc_id = var.vpc_id - +resource "aws_lb_target_group" "external_lb_tg_http" { + count = var.create_extlb ? 1 : 0 + port = var.extlb_listener_http_port + protocol = "TCP" + vpc_id = var.vpc_id + proxy_protocol_v2 = true depends_on = [ - aws_lb.external-lb + aws_lb.external_lb ] health_check { - protocol = "HTTP" - path = "/healthz" + protocol = "TCP" } lifecycle { @@ -59,60 +56,58 @@ resource "aws_lb_target_group" "external-lb-tg-http" { } tags = merge( - local.tags, + local.global_tags, { - Name = "lb-http-tg-${var.k8s_ext_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-ext-lb-tg-http-${var.environment}") } ) } -resource "aws_autoscaling_attachment" "target-http" { - count = var.install_nginx_ingress ? 1 : 0 +resource "aws_autoscaling_attachment" "target_http" { + count = var.create_extlb ? 1 : 0 depends_on = [ aws_autoscaling_group.k8s_workers_asg, - aws_lb_target_group.external-lb-tg-http + aws_lb_target_group.external_lb_tg_http ] autoscaling_group_name = aws_autoscaling_group.k8s_workers_asg.name - lb_target_group_arn = aws_lb_target_group.external-lb-tg-http[count.index].arn + lb_target_group_arn = aws_lb_target_group.external_lb_tg_http[count.index].arn } # HTTPS -resource "aws_lb_listener" "external-lb-listener-https" { - count = var.install_nginx_ingress ? 1 : 0 - load_balancer_arn = aws_lb.external-lb[count.index].arn +resource "aws_lb_listener" "external_lb_listener_https" { + count = var.create_extlb ? 1 : 0 + load_balancer_arn = aws_lb.external_lb[count.index].arn - protocol = "HTTPS" - port = var.extlb_https_port - certificate_arn = aws_acm_certificate.cert[count.index].arn + protocol = "TCP" + port = var.extlb_https_port default_action { type = "forward" - target_group_arn = aws_lb_target_group.external-lb-tg-https[count.index].arn + target_group_arn = aws_lb_target_group.external_lb_tg_https[count.index].arn } tags = merge( - local.tags, + local.global_tags, { - Name = "lb-https-listener-${var.k8s_ext_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-https-listener-${var.environment}") } ) } -resource "aws_lb_target_group" "external-lb-tg-https" { - count = var.install_nginx_ingress ? 1 : 0 - port = var.extlb_listener_https_port - protocol = "HTTPS" - vpc_id = var.vpc_id - +resource "aws_lb_target_group" "external_lb_tg_https" { + count = var.create_extlb ? 1 : 0 + port = var.extlb_listener_https_port + protocol = "TCP" + vpc_id = var.vpc_id + proxy_protocol_v2 = true depends_on = [ - aws_lb.external-lb + aws_lb.external_lb ] health_check { - protocol = "HTTPS" - path = "/healthz" + protocol = "TCP" } lifecycle { @@ -120,20 +115,79 @@ resource "aws_lb_target_group" "external-lb-tg-https" { } tags = merge( - local.tags, + local.global_tags, { - Name = "lb-https-tg-${var.k8s_ext_lb_name}-${var.environment}" + "Name" = lower("${var.common_prefix}-ext-lb-tg-https-${var.environment}") } ) } -resource "aws_autoscaling_attachment" "target-https" { - count = var.install_nginx_ingress ? 1 : 0 +resource "aws_autoscaling_attachment" "target_https" { + count = var.create_extlb ? 1 : 0 depends_on = [ aws_autoscaling_group.k8s_workers_asg, - aws_lb_target_group.external-lb-tg-https + aws_lb_target_group.external_lb_tg_https ] autoscaling_group_name = aws_autoscaling_group.k8s_workers_asg.name - lb_target_group_arn = aws_lb_target_group.external-lb-tg-https[count.index].arn + lb_target_group_arn = aws_lb_target_group.external_lb_tg_https[count.index].arn +} + +# kubeapi + +resource "aws_lb_listener" "external_lb_listener_kubeapi" { + count = var.expose_kubeapi ? 1 : 0 + load_balancer_arn = aws_lb.external_lb[count.index].arn + + protocol = "TCP" + port = var.kube_api_port + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.external_lb_tg_kubeapi[count.index].arn + } + + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-kubeapi-listener-${var.environment}") + } + ) +} + +resource "aws_lb_target_group" "external_lb_tg_kubeapi" { + count = var.expose_kubeapi ? 1 : 0 + port = var.kube_api_port + protocol = "TCP" + vpc_id = var.vpc_id + + depends_on = [ + aws_lb.external_lb + ] + + health_check { + protocol = "TCP" + } + + lifecycle { + create_before_destroy = true + } + + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-ext-lb-tg-kubeapi-${var.environment}") + } + ) +} + +resource "aws_autoscaling_attachment" "target_kubeapi" { + count = var.expose_kubeapi ? 1 : 0 + depends_on = [ + aws_autoscaling_group.k8s_servers_asg, + aws_lb_target_group.external_lb_tg_kubeapi + ] + + autoscaling_group_name = aws_autoscaling_group.k8s_servers_asg.name + lb_target_group_arn = aws_lb_target_group.external_lb_tg_kubeapi[count.index].arn } \ No newline at end of file diff --git a/lb_cert.tf b/lb_cert.tf deleted file mode 100644 index 5a1c2bc..0000000 --- a/lb_cert.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "aws_acm_certificate" "cert" { - count = var.install_nginx_ingress ? 1 : 0 - private_key = file(var.PATH_TO_PUBLIC_LB_KEY) - certificate_body = file(var.PATH_TO_PUBLIC_LB_CERT) -} \ No newline at end of file diff --git a/local.tf b/local.tf index 42c3b9e..284e228 100644 --- a/local.tf +++ b/local.tf @@ -1,8 +1,14 @@ locals { - tags = { - "environment" = "${var.environment}" - "provisioner" = "terraform" - "scope" = "k8s-cluster" - "uuid" = "${var.uuid}" + k8s_tls_san_public = var.create_extlb && var.expose_kubeapi ? aws_lb.external_lb[0].dns_name : "" + kubeconfig_secret_name = "${var.common_prefix}-kubeconfig/${var.cluster_name}/${var.environment}/v1" + kubeadm_ca_secret_name = "${var.common_prefix}-kubeadm-ca/${var.cluster_name}/${var.environment}/v1" + kubeadm_token_secret_name = "${var.common_prefix}-kubeadm-token/${var.cluster_name}/${var.environment}/v1" + kubeadm_cert_secret_name = "${var.common_prefix}-kubeadm-secret/${var.cluster_name}/${var.environment}/v1" + global_tags = { + environment = "${var.environment}" + provisioner = "terraform" + terraform_module = "https://github.com/garutilorenzo/k8s-aws-terraform-cluster" + k8s_cluster_name = "${var.cluster_name}" + application = "k8s" } } \ No newline at end of file diff --git a/output.tf b/output.tf index d33d24c..e804383 100644 --- a/output.tf +++ b/output.tf @@ -1,5 +1,5 @@ output "k8s_dns_name" { - value = aws_lb.external-lb.dns_name + value = var.create_extlb ? aws_lb.external_lb.*.dns_name : [] } output "k8s_server_private_ips" { diff --git a/secrets.tf b/secrets.tf new file mode 100644 index 0000000..fb3c583 --- /dev/null +++ b/secrets.tf @@ -0,0 +1,99 @@ +resource "aws_secretsmanager_secret" "kubeconfig_secret" { + name = local.kubeconfig_secret_name + description = "Kubeconfig k8s. Cluster name: ${var.cluster_name}, environment: ${var.environment}" + + tags = merge( + local.global_tags, + { + "Name" = lower("${local.kubeconfig_secret_name}") + } + ) +} + +resource "aws_secretsmanager_secret" "kubeadm_ca" { + name = local.kubeadm_ca_secret_name + description = "Kubeadm CA. Cluster name: ${var.cluster_name}, environment: ${var.environment}" + + tags = merge( + local.global_tags, + { + "Name" = lower("${local.kubeadm_ca_secret_name}") + } + ) +} + +resource "aws_secretsmanager_secret" "kubeadm_token" { + name = local.kubeadm_token_secret_name + description = "Kubeadm token. Cluster name: ${var.cluster_name}, environment: ${var.environment}" + + tags = merge( + local.global_tags, + { + "Name" = lower("${local.kubeadm_token_secret_name}") + } + ) +} + +resource "aws_secretsmanager_secret" "kubeadm_cert" { + name = local.kubeadm_cert_secret_name + description = "Kubeadm cert. Cluster name: ${var.cluster_name}, environment: ${var.environment}" + + tags = merge( + local.global_tags, + { + "Name" = lower("${local.kubeadm_cert_secret_name}") + } + ) +} + +# secret default values + +resource "aws_secretsmanager_secret_version" "kubeconfig_secret_default" { + secret_id = aws_secretsmanager_secret.kubeconfig_secret.id + secret_string = var.default_secret_placeholder +} + +resource "aws_secretsmanager_secret_version" "kubeadm_ca_default" { + secret_id = aws_secretsmanager_secret.kubeadm_ca.id + secret_string = var.default_secret_placeholder +} + +resource "aws_secretsmanager_secret_version" "kubeadm_token_default" { + secret_id = aws_secretsmanager_secret.kubeadm_token.id + secret_string = var.default_secret_placeholder +} + +resource "aws_secretsmanager_secret_version" "kubeadm_cert_default" { + secret_id = aws_secretsmanager_secret.kubeadm_cert.id + secret_string = var.default_secret_placeholder +} + +# Secret Policies + +resource "aws_secretsmanager_secret_policy" "kubeconfig_secret_policy" { + secret_arn = aws_secretsmanager_secret.kubeconfig_secret.arn + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + AWS = "${aws_iam_role.k8s_iam_role.arn}" + }, + Action = [ + "secretsmanager:GetSecretValue", + "secretsmanager:UpdateSecret", + "secretsmanager:DeleteSecret", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecrets", + "secretsmanager:CreateSecret", + "secretsmanager:PutSecretValue" + ] + Resource = [ + "${aws_secretsmanager_secret.kubeconfig_secret.arn}" + ] + } + ] + }) +} \ No newline at end of file diff --git a/security.tf b/security.tf index ab86a61..927e516 100644 --- a/security.tf +++ b/security.tf @@ -1,67 +1,91 @@ -resource "aws_security_group" "k8s-sg" { +resource "aws_security_group" "k8s_sg" { vpc_id = var.vpc_id - name = "k8s-sg" + name = "k8s_sg" description = "Kubernetes ingress rules" lifecycle { create_before_destroy = true } - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } + tags = merge( + local.global_tags, + { + "Name" = lower("${var.common_prefix}-allow-strict-${var.environment}") + } + ) +} - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = [var.vpc_subnet_cidr] - } +resource "aws_security_group_rule" "ingress_self" { + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + security_group_id = aws_security_group.k8s_sg.id +} - ingress { - from_port = var.kube_api_port - to_port = var.kube_api_port - protocol = "tcp" - cidr_blocks = [var.vpc_subnet_cidr] - } +resource "aws_security_group_rule" "ingress_kubeapi" { + type = "ingress" + from_port = var.kube_api_port + to_port = var.kube_api_port + protocol = "tcp" + cidr_blocks = [var.vpc_subnet_cidr] + security_group_id = aws_security_group.k8s_sg.id +} - ingress { - from_port = var.extlb_listener_http_port - to_port = var.extlb_listener_http_port - protocol = "tcp" - cidr_blocks = [var.vpc_subnet_cidr] - } +resource "aws_security_group_rule" "ingress_ssh" { + type = "ingress" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [var.my_public_ip_cidr] + security_group_id = aws_security_group.k8s_sg.id +} - ingress { - from_port = var.extlb_listener_https_port - to_port = var.extlb_listener_https_port - protocol = "tcp" - cidr_blocks = [var.vpc_subnet_cidr] - } +resource "aws_security_group_rule" "egress_all" { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.k8s_sg.id +} - ingress { - protocol = "-1" - self = true - from_port = 0 - to_port = 0 - } +resource "aws_security_group_rule" "allow_lb_http_traffic" { + count = var.create_extlb ? 1 : 0 + type = "ingress" + from_port = var.extlb_listener_http_port + to_port = var.extlb_listener_http_port + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.k8s_sg.id +} - tags = merge( - local.tags, - { - Name = "sg-k8s-cluster-${var.environment}" - } - ) +resource "aws_security_group_rule" "allow_lb_https_traffic" { + count = var.create_extlb ? 1 : 0 + type = "ingress" + from_port = var.extlb_listener_https_port + to_port = var.extlb_listener_https_port + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.k8s_sg.id +} + +resource "aws_security_group_rule" "allow_lb_kubeapi_traffic" { + count = var.create_extlb && var.expose_kubeapi ? 1 : 0 + type = "ingress" + from_port = var.kube_api_port + to_port = var.kube_api_port + protocol = "tcp" + cidr_blocks = [var.my_public_ip_cidr] + security_group_id = aws_security_group.k8s_sg.id } -resource "aws_security_group" "k8s-public-lb" { - count = var.install_nginx_ingress ? 1 : 0 +resource "aws_security_group" "efs_sg" { + count = var.efs_persistent_storage ? 1 : 0 vpc_id = var.vpc_id - name = "k8s-public-lb" - description = "Kubernetes public LB ingress rules" + name = "${var.common_prefix}-efs-sg-${var.environment}" + description = "Allow EFS access from VPC subnets" egress { from_port = 0 @@ -71,23 +95,16 @@ resource "aws_security_group" "k8s-public-lb" { } ingress { - from_port = var.extlb_http_port - to_port = var.extlb_http_port + from_port = 2049 + to_port = 2049 protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - from_port = var.extlb_https_port - to_port = var.extlb_https_port - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] + cidr_blocks = [var.vpc_subnet_cidr] } tags = merge( - local.tags, + local.global_tags, { - Name = "k8s-cluster-public-lb${var.environment}" + "Name" = lower("${var.common_prefix}-efs-sg-${var.environment}") } ) } \ No newline at end of file diff --git a/template.tf b/template.tf index 2be2dd8..821dce5 100644 --- a/template.tf +++ b/template.tf @@ -1,5 +1,5 @@ resource "aws_launch_template" "k8s_server" { - name_prefix = var.k8s_master_template_prefix + name_prefix = "${var.common_prefix}-server-tpl-${var.environment}" image_id = var.ami instance_type = var.default_instance_type user_data = data.template_cloudinit_config.k8s_server.rendered @@ -25,19 +25,19 @@ resource "aws_launch_template" "k8s_server" { network_interfaces { associate_public_ip_address = var.ec2_associate_public_ip_address - security_groups = [aws_security_group.k8s-sg.id] + security_groups = [aws_security_group.k8s_sg.id] } tags = merge( - local.tags, + local.global_tags, { - Name = "template-${var.k8s_master_template_prefix}-${var.environment}" + "Name" = lower("${var.common_prefix}-server-tpl-${var.environment}") } ) } resource "aws_launch_template" "k8s_worker" { - name_prefix = var.k8s_worker_template_prefix + name_prefix = "${var.common_prefix}-worker-tpl-${var.environment}" image_id = var.ami instance_type = var.default_instance_type user_data = data.template_cloudinit_config.k8s_worker.rendered @@ -63,13 +63,13 @@ resource "aws_launch_template" "k8s_worker" { network_interfaces { associate_public_ip_address = var.ec2_associate_public_ip_address - security_groups = [aws_security_group.k8s-sg.id] + security_groups = [aws_security_group.k8s_sg.id] } tags = merge( - local.tags, + local.global_tags, { - Name = "template-${var.k8s_worker_template_prefix}-${var.environment}" + "Name" = lower("${var.common_prefix}-worker-tpl-${var.environment}") } ) } \ No newline at end of file diff --git a/vars.tf b/vars.tf index ff98656..b8a11d7 100644 --- a/vars.tf +++ b/vars.tf @@ -2,10 +2,6 @@ variable "environment" { type = string } -variable "uuid" { - type = string -} - variable "ssk_key_pair_name" { type = string } @@ -15,6 +11,11 @@ variable "vpc_id" { description = "The vpc id" } +variable "my_public_ip_cidr" { + type = string + description = "My public ip CIDR" +} + variable "vpc_private_subnets" { type = list(any) description = "The private vpc subnets ids" @@ -30,60 +31,49 @@ variable "vpc_subnet_cidr" { description = "VPC subnet CIDR" } +variable "common_prefix" { + type = string + description = "" + default = "k8s" +} + variable "ec2_associate_public_ip_address" { type = bool default = false } -variable "s3_bucket_name" { - type = string - default = "my-very-secure-k8s-bucket" -} - -variable "instance_profile_name" { - type = string - default = "K8sInstanceProfile" -} +## eu-west-1 +# Ubuntu 22.04 +# ami-0cffefff2d52e0a23 -variable "iam_role_name" { - type = string - default = "K8sIamRole" -} +# Amazon Linux 2 AMI (HVM) - Kernel 5.10, SSD Volume Type +# ami-006be9ab6a140de6e variable "ami" { type = string - default = "ami-0a2616929f1e63d91" + default = " ami-0cffefff2d52e0a23" } variable "default_instance_type" { - type = string - default = "t3.large" + type = string + default = "t3.medium" + description = "Instance type to be used" } variable "instance_types" { description = "List of instance types to use" type = map(string) default = { - asg_instance_type_1 = "t3.large" - asg_instance_type_2 = "t2.large" - asg_instance_type_3 = "m4.large" - asg_instance_type_4 = "t3a.large" + asg_instance_type_1 = "t3.medium" + asg_instance_type_2 = "t3a.medium" + asg_instance_type_3 = "c5a.large" + asg_instance_type_4 = "c6a.large" } } -variable "k8s_master_template_prefix" { - type = string - default = "k8s_master_tpl" -} - -variable "k8s_worker_template_prefix" { - type = string - default = "k8s_worker_tpl" -} - variable "k8s_version" { type = string - default = "1.23.5" + default = "1.27.3" } variable "k8s_pod_subnet" { @@ -107,11 +97,6 @@ variable "kube_api_port" { description = "Kubeapi Port" } -variable "k8s_internal_lb_name" { - type = string - default = "k8s-server-tcp-lb" -} - variable "k8s_server_desired_capacity" { type = number default = 3 @@ -154,36 +139,46 @@ variable "cluster_name" { description = "Cluster name" } - -variable "PATH_TO_PUBLIC_LB_CERT" { - type = string - description = "Path to the public LB https certificate" +variable "install_nginx_ingress" { + type = bool + default = false + description = "Create external LB true/false" } -variable "PATH_TO_PUBLIC_LB_KEY" { - type = string - description = "Path to the public LB key" +variable "nginx_ingress_release" { + type = string + default = "v1.8.1" } -variable "install_longhorn" { +variable "install_certmanager" { type = bool default = false } -variable "longhorn_release" { +variable "certmanager_release" { type = string - default = "v1.2.3" + default = "v1.12.2" } -variable "install_nginx_ingress" { +variable "certmanager_email_address" { + type = string + default = "changeme@example.com" +} + +variable "create_extlb" { type = bool default = false description = "Create external LB true/false" } -variable "k8s_ext_lb_name" { +variable "efs_persistent_storage" { + type = bool + default = false +} + +variable "efs_csi_driver_release" { type = string - default = "k8s-ext-lb" + default = "v1.5.8" } variable "extlb_listener_http_port" { @@ -204,4 +199,24 @@ variable "extlb_http_port" { variable "extlb_https_port" { type = number default = 443 +} + +variable "default_secret_placeholder" { + type = string + default = "DEFAULTPLACEHOLDER" +} + +variable "expose_kubeapi" { + type = bool + default = false +} + +variable "install_node_termination_handler" { + type = bool + default = true +} + +variable "node_termination_handler_release" { + type = string + default = "v1.20.0" } \ No newline at end of file