diff --git a/control-plane/roles/gardener-operator/README.md b/control-plane/roles/gardener-operator/README.md new file mode 100644 index 00000000..f31d2d53 --- /dev/null +++ b/control-plane/roles/gardener-operator/README.md @@ -0,0 +1,168 @@ +# gardener-operator + +Deploys the Gardener Operator. + +Please refer to the metal-stack gardener integration in our [documentation](https://docs.metal-stack.io/stable/overview/kubernetes/). + +Check out the Gardener project for further documentation on [gardener.cloud](https://gardener.cloud/). + +## Variables + +| Name | Mandatory | Description | +| ------------------------------------------------------ | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| gardener_image_vector_overwrite | | Allows overriding the image vector to set custom image versions for gardener | +| gardener_component_image_vector_overwrite | | Allows overriding the image vector to set custom image versions for gardenlet components | +| gardener_apiserver_replicas | | Specifies the amount of gardener-apiserver replicas | +| gardener_apiserver_vpa | | Enables the VPA for the gardener-apiserver | +| gardener_apiserver_resources | | Set custom resource definitions for the gardener-apiserver | +| gardener_apiserver_feature_gates | | Sets features gates for the gardener-apiserver | +| gardener_apiserver_shoot_kubeconfig_max_expiration | | Max shoot kubeconfig expiration for the gardener-apiserver | +| gardener_controller_manager_resources | | Set custom resource definitions for the gardener-controller-manager | +| gardener_scheduler_resources | | Set custom resource definitions for the gardener-scheduler | +| gardener_dns_domain | | Specifies the DNS domain on which the Gardener will manage DNS entries | +| gardener_dns_provider | yes | Specifies the DNS provider | +| gardener_backup_infrastructure | | Specifies the Gardener backup infrastructure, required when `gardener_backup_infrastructure_secret` is set | +| gardener_backup_infrastructure_secret | | Specifies the secret for the backup infrastructure | +| gardener_soil_name | | The name of the initial `Seed` (used for spinning up shooted seeds) | +| gardener_soil_kubeconfig_file_path | | The kubeconfig path to the initial seed cluster | +| gardener_soil_vertical_pod_autoscaler_enabled | | Enables the VPA for the initial seed cluster | +| gardener_soil_project_owner_name | | Specifies the owner name for the project that the initial seed uses to set up shooted seeds | +| gardener_soil_project_members | | Specifies the members of the soil project. Each member requires a `name` and a `role`. Optionally, and array of `roles` can be specified. Example: `{"name": "admin", "role": "admin", "roles": ["owner"]}` | +| gardener_gardenlet_shoot_concurrent_syncs | | Specifies the amount of concurrent shoot syncs for the Gardenlet | +| gardener_gardenlet_shoot_reconcile_in_maintenance_only | | Specifies whether to reconcile shoots only in their maintenance time windows for the Gardenlet | +| gardener_gardenlet_shoot_respect_sync_period_overwrite | | Specifies whether to allow sync period overwrites for shoot resources | +| gardener_shooted_seeds | | A list of definitions for shooted seeds reconcile by the initial seed cluster, will be turned into `ManagedSeeds` | +| gardener_shooted_seed_max_pods | | The max pods amount for the shooted seeds | +| gardener_shooted_seed_node_cidr_mask_size | | The node CIDR mask size used for the kubelets of the shooted seeds | +| gardener_shooted_seed_rollout_delay_minutes | | An optional delay between shooted seed rollouts (can be used to calm down bigger environments during an update) | +| gardener_kube_api_server_kubeconfig | | The kubeconfig for the Gardener Kubernetes API (virtual garden apiserver) | +| gardener_kube_apiserver_kubeconfig_path | | The acts on multiple Kubernetes APIs, this is where it puts the kubeconfig of the Gardener Kubernetes API | +| gardener_local_tmp_dir | | The acts on multiple Kubernetes APIs, this is a local folder in the deployment container to store the kubeconfigs (ephemeral) | +| gardener_logging_enabled | | Specifies whether the logging Gardener logging stack should be activated in the Gardenlet | + +### Virtual Garden + +These variables are related to spinning up the virtual garden, a dedicated kube-apiserver, kube-controller-manager and ETCD to host all Gardener resources. This one will have no worker nodes and cannot schedule pods. + +The deployment chart is taken from [garden-setup](https://github.com/gardener/garden-setup) and follows the same deployment approach. + +| Name | Mandatory | Description | +| ---------------------------------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| gardener_virtual_api_server_svc_cluster_ip_add | | An integer to "guess" a free IP for the service that allows the soil to internally communicate with the virtual garden | +| gardener_virtual_api_server_public_dns | | The DNS domain to reach the virtual garden API server on | +| gardener_virtual_api_server_healthcheck_static_token | yes | A static token for healthchecking the virtual garden API server | +| gardener_etcd_backup_schedule | | The backup schedule for the virtual garden ETCD | +| gardener_etcd_snapshot_period | | The snapshot period for the virtual garden ETCD | +| gardener_etcd_garbage_collection_period | | The priod for garbage collection for the virtual garden ETCD | +| gardener_etcd_resources | | Set custom resource definitions for the virtual garden ETCD | +| gardener_virtual_api_oidc_issuer_url | | [Corresponds to the `--oidc-issuer-url` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-issuer-url) in the Kubernetes API server configuration. | +| gardener_virtual_api_oidc_client_id | | [Corresponds to the `--oidc-client-id` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-client-id) in the Kubernetes API server configuration. | +| gardener_virtual_api_oidc_username_claim | | [Corresponds to the `--oidc-username-claim` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-username-claim) in the Kubernetes API server configuration. | +| gardener_virtual_api_oidc_username_prefix | | [Corresponds to the `--oidc-username-prefix` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-username-prefix) in the Kubernetes API server configuration. | +| gardener_virtual_api_oidc_groups_claim | | [Corresponds to the `--oidc-groups-claim` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-groups-claim) in the Kubernetes API server configuration. | +| gardener_virtual_api_oidc_groups_prefix | | [Corresponds to the `--oidc-groups-prefix` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-groups-prefix) in the Kubernetes API server configuration. | +| gardener_virtual_api_oidc_ca | | [Corresponds to the `--oidc-ca-file` flag](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#oidc-groups-prefix) in the Kubernetes API server configuration. | + +### Cloud Profile + +Variables for the metal-stack cloud profile. + +| Name | Mandatory | Description | +| ---------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------- | +| gardener_cloud_profile_stage_name | | The name of the metal-stack environment in the cloud profile | +| gardener_cloud_profile_metal_api_url | | The URL used by the Gardener to communicate with the metal-api | +| gardener_cloud_profile_metal_api_hmac | yes | The admin HMAC used by the Gardener to communicate with the metal-api | +| gardener_cloud_profile_machine_images | | The machine images available for shoots in the metal-api | +| gardener_cloud_profile_firewall_images | | The firewall images available for shoots in the metal-api | +| gardener_cloud_profile_firewall_images_from_machine_images | | If set to true, uses the passed machine images and adds those with firewall feature | +| gardener_cloud_profile_firewall_controller_versions | | The available firewall controller versions for metal-stack shoots | +| gardener_cloud_profile_kubernetes | | The available Kubernetes versions for metal-stack shoots | +| gardener_cloud_profile_machine_types | | The machine types available for shoots in the metal-api | +| gardener_cloud_profile_regions | | The regions available for shoots | +| gardener_cloud_profile_partitions | | The partitions available for shoots | +| gardener_os_cri_mapping | | A mapping to add available CRIs to the machine images | + +### Extensions + +These variable parametrize the Gardener extension controllers. + +This includes the metal-stack extension provider called [gardener-extension-provider-metal](https://github.com/metal-stack/gardener-extension-provider-metal) (GEPM). + +| Name | Mandatory | Description | +| ------------------------------------------------------------ | --------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| gardener_extension_provider_gcp_enabled | | If enabled, deploys the gardener-extension-provider-metal | +| gardener_extension_os_metal_enabled | | If enabled, deploys the os-metal-extension | +| gardener_extension_networking_calico_enabled | | If enabled, deploys the gardener-networking-extension-calico | +| gardener_extension_networking_cilium_enabled | | If enabled, deploys the gardener-networking-extension-cilium | +| gardener_extension_shoot_cert_service_enabled | | If enabled, deploys the gardener-extension-shoot-cert-service | +| gardener_extension_shoot_dns_service_enabled | | If enabled, deploys the gardener-extension-shoot-dns-service | +| gardener_extension_backup_s3_enabled | | If enabled, deploys the gardener-extension-backup-s3 | +| gardener_extension_dns_powerdns_enabled | | If enabled, deploys the gardener-extension-dns-powerdns | +| gardener_os_controller_repo_ref | | A repo reference for deploying the [os-metal-extension](https://github.com/metal-stack/os-metal-extension/) | +| gardener_networking_cilium_repo_ref | | A repo reference for deploying the [gardener-extension-networking-cilium](https://github.com/gardener/gardener-extension-networking-cilium) | +| gardener_extension_provider_metal_repo_ref | | A repo reference for deploying the [gardener-extension-provider-metal](https://github.com/metal-stack/gardener-extension-provider-metal) | +| gardener_shoot_dns_service_repo_ref | | A repo reference for deploying the [gardener-extension-shoot-dns-service](https://github.com/gardener/gardener-extension-shoot-dns-service) | +| gardener_extension_backup_s3_repo_ref | | A repo reference for deploying the [gardener-extension-backup-s3](https://github.com/metal-stack/gardener-extension-backup-s3) | +| gardener_extension_dns_powerdns_repo_ref | | A repo reference for deploying the [gardener-extension-dns-powerdns](https://github.com/metal-stack/gardener-extension-dns-powerdns) | +| gardener_metal_admission_replicas | | Specifies the amount of metal-admission webhook replicas | +| gardener_metal_admission_vpa | | Enables the VPA for the metal-admission webhook | +| gardener_extension_provider_metal_cluster_audit_enabled | | Enables the audit functionality of the GEPM | +| gardener_extension_provider_metal_audit_to_splunk_enabled | | Enables the audit to splunk feature gate of the GEPM | +| gardener_extension_provider_metal_audit_to_splunk | | Configuration for the audit to splunk feature gate of the GEPM | +| gardener_extension_provider_metal_etcd_storage_class_name | | The storage class used for metal-stack shoot ETCDs | +| gardener_extension_provider_metal_etcd_backup_schedule | | The backup schedule for metal-stack shoot ETCDs | +| gardener_extension_provider_metal_etcd_delta_snapshot_period | | The delta snapshot period for metal-stack shoot ETCDs | +| gardener_extension_provider_metal_egress_destinations | | Sets allowed egress destinations for the `RestrictEgress` control plane feature gate of the GEPM | +| gardener_extension_provider_metal_duros_storage_enabled | | Enables the duros storage integration feature gate of the GEPM (Lightbits storage) | +| gardener_extension_provider_metal_duros_storage_config | | Configuration for the duros storage integration | +| gardener_extension_provider_metal_image_pull_policy | | Sets the image pull policy for components deployed through this extension controller. | +| gardener_extension_provider_metal_image_pull_secret | | Provide image pull secrets for deployed containers | +| gardener_cert_management_issuer_private_key | | The Let's Encrypt private key used by the cert-management extension controller to setup signed certificates | +| gardener_extension_networking_cilium_image_vector_overwrite | | Allows overriding the image vector for the networking cilium extension | +| gardener_cert_management_issuer_email | | The issuer email used by the cert-management extension | +| gardener_cert_management_issuer_server | | The issuer server used by the cert-management extension | +| gardener_cert_management_precheck_nameservers | | To provide special set of nameservers to be used for prechecking DNSChallenges for an issuer | +| gardener_cert_management_shoot_issuers_enabled | | If enabled, allows to specify issuers in the shoot clusters | +| gardener_shoot_dns_service_image_vector_overwrite | | Allows overriding the image vector for the shoot-dns-service extension | +| gardener_shoot_dns_service_dns_controller_manager_image_name | | Setting an explicit image name for the dns-controller-manager | +| gardener_shoot_dns_service_dns_controller_manager_image_tag | | Setting an explicit image tag for the dns-controller-manager | +| gardener_extension_backup_s3_image_name | | Setting an explicit image name for the gardener-extension-backup-s3 | +| gardener_extension_backup_s3_image_tag | | Setting an explicit image tag for the gardener-extension-backup-s3 | +| gardener_extension_dns_powerdns_image_name | | Setting an explicit image name for the gardener-extension-dns-powerdns | +| gardener_extension_dns_powerdns_image_tag | | Setting an explicit image tag for the gardener-extension-dns-powerdns | + +### Certificates + +Gardener requires quite a lot of certificates, which should be self-signed and have to be generated before the deployment. + +We use a small shell script as in the [mini-lab](https://github.com/metal-stack/mini-lab/blob/master/files/certs/roll_certs.sh) to generate the certificates. + +| Name | Mandatory | Description | +| -------------------------------------------- | --------- | ----------- | +| gardener_kube_api_server_ca | yes | - | +| gardener_kube_api_server_ca_key | yes | - | +| gardener_kube_api_server_cert | yes | - | +| gardener_kube_api_server_key | yes | - | +| gardener_kube_api_server_client_cert | yes | - | +| gardener_kube_api_server_client_key | yes | - | +| gardener_kube_aggregator_client_cert | yes | - | +| gardener_kube_aggregator_client_key | yes | - | +| gardener_kube_controller_manager_client_cert | yes | - | +| gardener_kube_controller_manager_client_key | yes | - | +| gardener_admin_client_cert | yes | - | +| gardener_admin_client_key | yes | - | +| gardener_service_account_client_key | yes | - | +| gardener_api_server_ca | yes | - | +| gardener_api_server_cert | yes | - | +| gardener_api_server_key | yes | - | +| gardener_admission_controller_ca | yes | - | +| gardener_admission_controller_cert | yes | - | +| gardener_admission_controller_key | yes | - | +| gardener_controller_manager_ca | yes | - | +| gardener_controller_manager_cert | yes | - | +| gardener_controller_manager_key | yes | - | +| gardener_etcd_ca_cert | yes | - | +| gardener_etcd_cert | yes | - | +| gardener_etcd_cert_key | yes | - | +| gardener_etcd_client_cert | yes | - | +| gardener_etcd_client_key | yes | - | diff --git a/control-plane/roles/gardener/defaults/main/certs.yaml b/control-plane/roles/gardener-operator/defaults/main/certs.yaml similarity index 100% rename from control-plane/roles/gardener/defaults/main/certs.yaml rename to control-plane/roles/gardener-operator/defaults/main/certs.yaml diff --git a/control-plane/roles/gardener-operator/defaults/main/control-plane-defaults b/control-plane/roles/gardener-operator/defaults/main/control-plane-defaults new file mode 120000 index 00000000..a77f2709 --- /dev/null +++ b/control-plane/roles/gardener-operator/defaults/main/control-plane-defaults @@ -0,0 +1 @@ +../../../../control-plane-defaults/ \ No newline at end of file diff --git a/control-plane/roles/gardener-operator/defaults/main/gardener.yaml b/control-plane/roles/gardener-operator/defaults/main/gardener.yaml new file mode 100644 index 00000000..4a3a98ae --- /dev/null +++ b/control-plane/roles/gardener-operator/defaults/main/gardener.yaml @@ -0,0 +1,78 @@ +--- +gardener_image_vector_overwrite: +gardener_component_image_vector_overwrite: + +gardener_operator_enabled: false + +gardener_apiserver_replicas: 1 +gardener_apiserver_vpa: true +gardener_apiserver_feature_gates: {} +gardener_apiserver_shoot_kubeconfig_max_expiration: "8760h" + +gardener_apiserver_resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 400m + memory: 1Gi + +gardener_controller_manager_resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 750m + memory: 512Mi + +gardener_scheduler_resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 300m + memory: 256Mi + +gardener_dns_domain: +gardener_dns_provider: + +gardener_backup_infrastructure: + provider: local + bucket: gardener-operator + + # provider: gcp + # region: + # secretRef: + # name: backup-secret + # namespace: garden + # bucket: + # + # provider: S3 + # endpoint: "{{ gardener_backup_infrastructure_secret.endpoint | b64decode }}" + # accessKeyID: "{{ gardener_backup_infrastructure_secret.accessKeyID | b64decode }}" + # secretAccessKey: "{{ gardener_backup_infrastructure_secret.secretAccessKey | b64decode}}" + +gardener_backup_infrastructure_secret: + hostPath: "{{ '/etc/gardener/local-backupbuckets' | b64encode }}" + + # for gcp: + # serviceaccount.json: "{{ gardener_backup_infrastructure_service_account_json | b64encode }}" + # + # for S3: + # endpoint: + # accessKeyID: + # secretAccessKey: + +gardener_soil_name: "{{ metal_control_plane_stage_name }}" +gardener_soil_kubeconfig_file_path: "{{ lookup('env', 'KUBECONFIG') }}" +gardener_soil_vertical_pod_autoscaler_enabled: false +gardener_soil_project_owner_name: admin +gardener_soil_project_members: [] + +gardener_gardenlet_shoot_concurrent_syncs: 20 +gardener_gardenlet_shoot_reconcile_in_maintenance_only: false +gardener_gardenlet_shoot_respect_sync_period_overwrite: true + +gardener_local_tmp_dir: "{{ playbook_dir }}/.ansible/tmp" + +gardener_logging_enabled: false diff --git a/control-plane/roles/gardener-operator/defaults/main/global-defaults b/control-plane/roles/gardener-operator/defaults/main/global-defaults new file mode 120000 index 00000000..ac0cbf4d --- /dev/null +++ b/control-plane/roles/gardener-operator/defaults/main/global-defaults @@ -0,0 +1 @@ +../../../../../defaults \ No newline at end of file diff --git a/control-plane/roles/gardener/defaults/main/operator.yaml b/control-plane/roles/gardener-operator/defaults/main/operator.yaml similarity index 100% rename from control-plane/roles/gardener/defaults/main/operator.yaml rename to control-plane/roles/gardener-operator/defaults/main/operator.yaml diff --git a/control-plane/roles/gardener/defaults/main/virtual_garden.yaml b/control-plane/roles/gardener-operator/defaults/main/virtual_garden.yaml similarity index 100% rename from control-plane/roles/gardener/defaults/main/virtual_garden.yaml rename to control-plane/roles/gardener-operator/defaults/main/virtual_garden.yaml diff --git a/control-plane/roles/gardener-operator/filter_plugins/common.py b/control-plane/roles/gardener-operator/filter_plugins/common.py new file mode 100644 index 00000000..50c6f671 --- /dev/null +++ b/control-plane/roles/gardener-operator/filter_plugins/common.py @@ -0,0 +1,286 @@ +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +import base64 +import ipaddress +import yaml + +from ansible.module_utils.six import PY3 +from ansible.plugins.test.core import version_compare +from ansible.errors import AnsibleFilterError + + +def b64decode(source): + content = base64.b64decode(source) + if PY3: + content = content.decode('utf-8') + return content + + +def b64encode(source): + if PY3: + source = source.encode('utf-8') + content = base64.b64encode(source) + if PY3: + content = content.decode('utf-8') + return content + + +def _extract_cluster_from_kubeconfig(kubeconfig_path): + with open(kubeconfig_path, 'r') as f: + kubeconfig = yaml.safe_load(f) + + current_context_name = kubeconfig['current-context'] + contexts = kubeconfig.get('contexts', []) + current_contexts = [context for context in contexts if context["name"] == current_context_name] + + if not current_contexts: + raise AnsibleFilterError("current context not found in kubeconfig") + + current_context = current_contexts[0] + cluster_name = current_context.get("context", dict()).get("cluster") + + if not cluster_name: + raise AnsibleFilterError("cluster name not defined in current context") + + clusters = kubeconfig.get('clusters', []) + current_clusters = [cluster for cluster in clusters if cluster["name"] == cluster_name] + + if not current_clusters: + raise AnsibleFilterError("current cluster not found in kubeconfig") + + return current_clusters[0].get("cluster", dict()) + + +def kubeconfig_for_sa(kubeconfig_path, secret): + cluster = _extract_cluster_from_kubeconfig(kubeconfig_path) + + server = cluster.get("server") + + secret_data = secret.get("data", dict()) + ca = str(secret_data.get("ca.crt")) + token = b64decode(secret_data.get("token")) + namespace = b64decode(secret_data.get("namespace")) + + return yaml.safe_dump({ + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": ca, + "server": server, + } + } + ], + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "namespace": namespace, + "user": "default-user", + } + } + ], + "current-context": "default-context", + "users": [ + { + "name": "default-user", + "user": { + "token": token, + } + } + ], + }) + + +def extract_gcp_node_network(subnets, region): + for subnet in subnets: + subnetwork = subnet.get("subnetwork") + if not subnetwork: + continue + + if region in subnetwork: + return subnet.get("ipCidrRange") + + raise AnsibleFilterError("no node network found for region: %s" % region) + + +def managed_seed_annotation(managed_seed_api_server, api_server=None): + if not managed_seed_api_server: + return "" + + settings = [] + + if api_server: + defaultReplicas = api_server.get("replicas") + if defaultReplicas: + settings.append("apiServer.replicas=" + str(defaultReplicas)) + + autoscaler = api_server.get("autoscaler", dict()) + + min_replicas = autoscaler.get("min_replicas") + if min_replicas: + settings.append("apiServer.autoscaler.minReplicas=" + str(min_replicas)) + + max_replicas = autoscaler.get("max_replicas") + if max_replicas: + settings.append("apiServer.autoscaler.maxReplicas=" + str(max_replicas)) + + return ",".join(settings) + + +def network_cidr_add(cidr, add): + return str(ipaddress.ip_network(cidr).network_address + add) + + +def kubeconfig_from_cert(server, ca, cert, key, prepend_https=False): + if prepend_https and not server.startswith("https"): + server = "https://" + server + + return yaml.safe_dump({ + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": b64encode(ca), + "server": server, + } + } + ], + "current-context": "default-context", + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "user": "default-user", + } + } + ], + "users": [ + { + "name": "default-user", + "user": { + "client-certificate-data": b64encode(cert), + "client-key-data": b64encode(key), + } + } + ], + }) + + +def kubeconfig_from_token(server, ca, token, prepend_https=False): + if prepend_https and not server.startswith("https"): + server = "https://" + server + + return yaml.safe_dump({ + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": b64encode(ca), + "server": server, + } + } + ], + "current-context": "default-context", + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "user": "default-user", + } + } + ], + "users": [ + { + "name": "default-user", + "user": { + "token": token, + } + } + ], + }) + + +def machine_images_for_cloud_profile(image_list, cris=None): + images = dict() + for image in image_list: + if 'machine' not in image.get("features", list()): + continue + + if image.get('omit_from_cloud_profile', False): + continue + + image_id = image.get("id") + if image_id is None: + continue + + parts = image_id.split("-") + name = "-".join(parts[:-1]) + + version = parts[-1] + + version_parts = version.split(".") + # ubuntu-19.10.20200331 + # major = version_parts[0] + minor = ".".join(version_parts[:2]) + + image_versions = images.get(name, set()) + # Do not add the major version to the vector + # metal-api cannot match latest version if only major is given + # image_versions.add(major) + image_versions.add(minor) + image_versions.add(version) + images[name] = image_versions + + result = list() + for name, value in images.items(): + versions = list() + for v in sorted(list(value)): + version = dict( + version=v + ) + + if cris is not None and name in cris: + cri = cris[name].copy() + cri_config = cri.pop("cris", []) + cri_condition = cri.pop("when", None) + + if cri_condition is None: + version["cri"] = cri_config + else: + if version_compare(v, cri_condition["version"], cri_condition["operator"]): + version["cri"] = cri_config + + versions.append(version) + + image = dict( + name=name, + versions=versions, + ) + result.append(image) + + return result + + +class FilterModule(object): + def filters(self): + return { + 'network_cidr_add': network_cidr_add, + 'kubeconfig_from_cert': kubeconfig_from_cert, + 'kubeconfig_from_token': kubeconfig_from_token, + 'machine_images_for_cloud_profile': machine_images_for_cloud_profile, + 'kubeconfig_for_sa': kubeconfig_for_sa, + 'extract_gcp_node_network': extract_gcp_node_network, + 'managed_seed_annotation': managed_seed_annotation, + } diff --git a/control-plane/roles/gardener-operator/tasks/access_kubeconfig.yaml b/control-plane/roles/gardener-operator/tasks/access_kubeconfig.yaml new file mode 100644 index 00000000..9e6015f0 --- /dev/null +++ b/control-plane/roles/gardener-operator/tasks/access_kubeconfig.yaml @@ -0,0 +1,67 @@ +--- + +- name: Create virtual garden access secret + k8s: + apply: yes + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: shoot-access-virtual-garden + namespace: garden + labels: + resources.gardener.cloud/purpose: token-requestor + resources.gardener.cloud/class: shoot + annotations: + serviceaccount.resources.gardener.cloud/name: virtual-garden-user + serviceaccount.resources.gardener.cloud/namespace: kube-system + serviceaccount.resources.gardener.cloud/token-expiration-duration: 3h + +- name: Create virtual garden access managed resource secret + k8s: + apply: yes + definition: + apiVersion: v1 + kind: Secret + metadata: + name: managedresource-virtual-garden-access + namespace: garden + type: Opaque + stringData: + clusterrolebinding____gardener.cloud.virtual-garden-access.yaml: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: gardener.cloud.sap:virtual-garden + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: virtual-garden-user + namespace: kube-system + +- name: Create virtual garden access managed resource + k8s: + apply: yes + definition: + apiVersion: resources.gardener.cloud/v1alpha1 + kind: ManagedResource + metadata: + name: virtual-garden-access + namespace: garden + spec: + secretRefs: + - name: managedresource-virtual-garden-access + +- name: Create kubeconfig for accessing the virtual garden + set_fact: + gardener_kube_api_server_kubeconfig: "{{ (gardener_virtual_api_server_public_dns + ':' + gardener_virtual_api_server_public_wait_port | string) | kubeconfig_from_token(gardener_kube_api_server_ca, lookup('k8s', api_version='v1', kind='Secret', namespace='garden', resource_name='shoot-access-virtual-garden').get('data', {}).get('token') | b64decode, prepend_https=true) }}" + +- name: Wait for garden-kube-apiserver + wait_for: + host: "{{ gardener_virtual_api_server_public_dns }}" + port: "{{ gardener_virtual_api_server_public_wait_port }}" + timeout: 60 diff --git a/control-plane/roles/gardener-operator/tasks/control-plane-networks/gcp.yaml b/control-plane/roles/gardener-operator/tasks/control-plane-networks/gcp.yaml new file mode 100644 index 00000000..c886eb75 --- /dev/null +++ b/control-plane/roles/gardener-operator/tasks/control-plane-networks/gcp.yaml @@ -0,0 +1,41 @@ +--- +# GKE sometimes changes the cidrs, which is a validation error for Gardener. +# Therefore, do not touch the cidrs once deployed +- name: Describe GCP cluster + command: gcloud --format json container clusters describe {{ gcp_cluster_name }} --region {{ gcp_region }} + register: gcp_cluster + changed_when: false + +- name: Describe GCP container subnets + command: gcloud --format json container subnets list-usable + register: gcp_subnets + changed_when: false + +# this config map provides the external api-server ip of the gke cluster. +# this information is unfortunately not provided by gke itself in the cluster. +# the ip is required by the gardener-extension-provider-metal to create a kubeconfig +# for the firewall-controller, which can then connect to the kube-apiserver for +# reconciling the firewall v2 resource. +- name: Deploy seed-api-server config map + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: seed-api-server + namespace: garden + data: + url: "https://{{ (gcp_cluster.stdout | from_json)['endpoint'] }}" + apply: yes + +- name: Set node cidr + set_fact: + _gardener_cluster_node_cidr: "{{ (gcp_subnets.stdout | from_json) | extract_gcp_node_network(region=gcp_region) }}" + +- name: Set pod cidr + set_fact: + _gardener_cluster_pod_cidr: "{{ (gcp_cluster.stdout | from_json)['ipAllocationPolicy']['clusterIpv4CidrBlock'] }}" + +- name: Set service cidr + set_fact: + _gardener_cluster_service_cidr: "{{ (gcp_cluster.stdout | from_json)['servicesIpv4Cidr'] }}" diff --git a/control-plane/roles/gardener-operator/tasks/control-plane-networks/metal.yaml b/control-plane/roles/gardener-operator/tasks/control-plane-networks/metal.yaml new file mode 100644 index 00000000..d97b3336 --- /dev/null +++ b/control-plane/roles/gardener-operator/tasks/control-plane-networks/metal.yaml @@ -0,0 +1,12 @@ +--- +- name: Set node cidr + set_fact: + _gardener_cluster_node_cidr: "{{ (lookup('k8s', api_version='v1', kind='ConfigMap', namespace='kube-system', resource_name='shoot-info') | default({}, true)).get('data', {})['nodeNetwork'] }}" + +- name: Set pod cidr + set_fact: + _gardener_cluster_pod_cidr: "{{ (lookup('k8s', api_version='v1', kind='ConfigMap', namespace='kube-system', resource_name='shoot-info') | default({}, true)).get('data', {})['podNetwork'] }}" + +- name: Set service cidr + set_fact: + _gardener_cluster_service_cidr: "{{ (lookup('k8s', api_version='v1', kind='ConfigMap', namespace='kube-system', resource_name='shoot-info') | default({}, true)).get('data', {})['serviceNetwork'] }}" diff --git a/control-plane/roles/gardener/tasks/local_backup.yaml b/control-plane/roles/gardener-operator/tasks/local_backup.yaml similarity index 100% rename from control-plane/roles/gardener/tasks/local_backup.yaml rename to control-plane/roles/gardener-operator/tasks/local_backup.yaml diff --git a/control-plane/roles/gardener-operator/tasks/main.yaml b/control-plane/roles/gardener-operator/tasks/main.yaml new file mode 100644 index 00000000..f34e1631 --- /dev/null +++ b/control-plane/roles/gardener-operator/tasks/main.yaml @@ -0,0 +1,111 @@ +--- +- name: Gather release versions + setup_yaml: + +- name: Check mandatory variables for this role are set + assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: yes + that: + - gardener_repo_url is defined + - gardener_repo_ref is defined + - gardener_gardenlet_image_tag is defined + - gardener_gardenlet_image_name is defined + - gardener_etcd_repo_ref is defined + - gardener_apiserver_image_tag is defined + - gardener_controller_manager_image_tag is defined + - gardener_scheduler_image_tag is defined + - gardener_extension_provider_metal_image_tag is defined + - gardener_os_controller_image_tag is defined + - metal_cloud_controller_manager_image_tag is defined + - gardener_networking_calico_image_tag is defined + - csi_lvm_controller_image_tag is defined + - csi_lvm_provisioner_image_tag is defined + - gardener_api_server_ca is not none + - gardener_api_server_ca_key is not none + - gardener_api_server_cert is not none + - gardener_api_server_key is not none + - gardener_api_server_client_cert is not none + - gardener_api_server_client_key is not none + - gardener_kube_aggregator_client_cert is not none + - gardener_kube_aggregator_client_key is not none + - gardener_kube_controller_manager_client_cert is not none + - gardener_kube_controller_manager_client_key is not none + - gardener_admin_client_cert is not none + - gardener_admin_client_key is not none + - gardener_service_account_client_key is not none + - gardener_admission_controller_ca is not none + - gardener_admission_controller_cert is not none + - gardener_admission_controller_key is not none + - gardener_controller_manager_ca is not none + - gardener_controller_manager_cert is not none + - gardener_controller_manager_key is not none + - gardener_admission_metal_ca is not none + - gardener_admission_metal_cert is not none + - gardener_admission_metal_key is not none + - gardener_etcd_ca_cert is not none + - gardener_etcd_cert is not none + - gardener_etcd_cert_key is not none + - gardener_etcd_client_cert is not none + - gardener_etcd_client_key is not none + - gardener_virtual_api_server_public_dns is not none + - gardener_virtual_api_server_healthcheck_static_token is not none + - gardener_dns_domain is not none + - gardener_dns_provider is not none + - gardener_cloud_profile_metal_api_url is not none + - gardener_cloud_profile_metal_api_hmac is not none + - gardener_backup_infrastructure is not none and gardener_backup_infrastructure.provider in ["local", "gcp", "S3"] + - gardener_cert_management_issuer_email is not none + +- name: Deploy required Seed CRDs + k8s: + definition: "{{ item.definition }}" + apply: yes + loop: + - name: hvpas + definition: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener/' + gardener_repo_ref + '/example/seed-crds/10-crd-autoscaling.k8s.io_hvpas.yaml', split_lines=False) | from_yaml_all | list)[0] }}" + - name: vpas + definition: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener/' + gardener_repo_ref + '/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml', split_lines=False) | from_yaml_all | list)[0] }}" + loop_control: + label: "{{ item.name }}" + +- name: Create garden namespace + k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: garden + +- name: Fail when provider unsupported + fail: + msg: "provider is unsupported: {{ metal_control_plane_host_provider }}" + when: + - metal_control_plane_host_provider not in ["gcp", "metal"] + +- name: GCP control plane service cidr + import_tasks: control-plane-networks/gcp.yaml + when: + - metal_control_plane_host_provider == "gcp" + +- name: metal control plane service cidr + import_tasks: control-plane-networks/metal.yaml + when: + - metal_control_plane_host_provider == "metal" + +- name: Create backup directory for local deployment + import_tasks: local_backup.yaml + when: gardener_backup_infrastructure.provider == "local" + +- name: Clone Gardener + git: + repo: "{{ gardener_repo_url }}" + dest: "{{ gardener_local_tmp_dir }}/gardener" + depth: 1 + version: "{{ gardener_repo_ref }}" + +- name: Deploy Gardener Operator + import_tasks: gardener_operator.yaml + +- name: Virtual Garden Access + import_tasks: access_kubeconfig.yaml diff --git a/control-plane/roles/gardener-operator/tasks/operator.yaml b/control-plane/roles/gardener-operator/tasks/operator.yaml new file mode 100644 index 00000000..3e3e80ef --- /dev/null +++ b/control-plane/roles/gardener-operator/tasks/operator.yaml @@ -0,0 +1,47 @@ +--- +- name: Create backup infrastructure secret + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: virtual-garden-etcd-main-backup-secret + namespace: garden + type: Opaque + data: "{{ gardener_backup_infrastructure_secret }}" + apply: yes + +# TODO: prepare migration here +# - label existing secrets like CA, ETCD encryption key, accordingly +# - scale down existing components (gardener control plane + virtual garden with ETCD) +# +# https://github.com/gardener/gardener/blob/v1.100.2/docs/concepts/operator.md#migrating-an-existing-gardener-landscape-to-gardener-operator + +- name: Deploy Gardener Operator + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_timeout: "600s" + helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/operator" + helm_release_name: operator + helm_target_namespace: garden + helm_value_file_template: gardener-operator-values.j2 + +- name: Create Garden + k8s: + definition: "{{ lookup('template', 'garden.yaml') }}" + apply: yes + +- name: Wait until Garden is ready + kubernetes.core.k8s_info: + api_version: "operator.gardener.cloud/v1alpha1" + kind: Garden + name: "local" + wait: yes + wait_condition: + status: "True" + type: "{{ item }}" + wait_timeout: 900 + loop: + - VirtualComponentsHealthy + - RuntimeComponentsHealthy diff --git a/control-plane/roles/gardener/templates/garden.yaml b/control-plane/roles/gardener-operator/templates/garden.yaml similarity index 98% rename from control-plane/roles/gardener/templates/garden.yaml rename to control-plane/roles/gardener-operator/templates/garden.yaml index a7eb8c3d..16c226c3 100644 --- a/control-plane/roles/gardener/templates/garden.yaml +++ b/control-plane/roles/gardener-operator/templates/garden.yaml @@ -13,9 +13,9 @@ spec: # providerConfig: # networking: - nodes: "{{ _gardener_soil_node_cidr }}" - pods: "{{ _gardener_soil_pod_cidr }}" - services: "{{ _gardener_soil_service_cidr }}" + nodes: "{{ _gardener_cluster_node_cidr }}" + pods: "{{ _gardener_cluster_pod_cidr }}" + services: "{{ _gardener_cluster_service_cidr }}" # provider: # zones: # - "0" diff --git a/control-plane/roles/gardener/templates/gardener-operator-values.j2 b/control-plane/roles/gardener-operator/templates/gardener-operator-values.j2 similarity index 100% rename from control-plane/roles/gardener/templates/gardener-operator-values.j2 rename to control-plane/roles/gardener-operator/templates/gardener-operator-values.j2 diff --git a/control-plane/roles/gardener/defaults/main/gardener.yaml b/control-plane/roles/gardener/defaults/main/gardener.yaml index 6096c204..02e85324 100644 --- a/control-plane/roles/gardener/defaults/main/gardener.yaml +++ b/control-plane/roles/gardener/defaults/main/gardener.yaml @@ -2,8 +2,6 @@ gardener_image_vector_overwrite: gardener_component_image_vector_overwrite: -gardener_operator_enabled: false - gardener_apiserver_replicas: 1 gardener_apiserver_vpa: true gardener_apiserver_feature_gates: {} diff --git a/control-plane/roles/gardener/files/kube-apiserver/Chart.yaml b/control-plane/roles/gardener/files/kube-apiserver/Chart.yaml deleted file mode 100644 index 54d65cde..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -description: Helm chart for garden -name: garden -version: 0.1.0 diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl b/control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl deleted file mode 100644 index eeb65337..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- define "garden.kubeconfig-controller-manager" -}} -apiVersion: v1 -kind: Config -current-context: garden -contexts: -- context: - cluster: garden - user: kube-controller-manager - name: garden -clusters: -- cluster: - certificate-authority-data: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - server: https://localhost:443 - name: garden -users: -- name: kube-controller-manager - user: - client-certificate-data: {{ .Values.tls.kubeControllerManager.crt | b64enc }} - client-key-data: {{ .Values.tls.kubeControllerManager.key | b64enc }} -{{- end -}} - -{{- define "garden.kubeconfig-gardener" -}} -apiVersion: v1 -kind: Config -current-context: garden -contexts: -- context: - cluster: garden - user: gardener - name: garden -clusters: -- cluster: - certificate-authority-data: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - server: https://{{ .Values.apiServer.serviceName }}:443 - name: garden -users: -- name: gardener - user: - client-certificate-data: {{ .Values.tls.gardener.crt | b64enc }} - client-key-data: {{ .Values.tls.gardener.key | b64enc }} -{{- end -}} - -{{- define "garden.kubeconfig-admin" -}} -apiVersion: v1 -kind: Config -current-context: garden -contexts: -- context: - cluster: garden - user: admin - name: garden -clusters: -- cluster: - certificate-authority-data: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - server: https://{{ .Values.apiServer.hostname }}:{{ .Values.apiServer.port }} - name: garden -users: -- name: admin - user: - client-certificate-data: {{ .Values.tls.admin.crt | b64enc }} - client-key-data: {{ .Values.tls.admin.key | b64enc }} -{{- end -}} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml deleted file mode 100644 index 69d1becd..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: garden-kube-apiserver - namespace: {{ .Release.Namespace }} - labels: - app: garden - component: kube-apiserver -spec: - revisionHistoryLimit: 0 - replicas: {{ .Values.replicas }} - selector: - matchLabels: - app: garden - component: kube-apiserver - template: - metadata: - annotations: - checksum/secret-kube-aggregator-ca: {{ include (print $.Template.BasePath "/secret-kube-aggregator-ca.yaml") . | sha256sum }} - checksum/secret-kube-aggregator-client: {{ include (print $.Template.BasePath "/secret-kube-aggregator-client-tls.yaml") . | sha256sum }} - checksum/secret-kube-apiserver-ca: {{ include (print $.Template.BasePath "/secret-kube-apiserver-ca.yaml") . | sha256sum }} - checksum/secret-kube-apiserver-server: {{ include (print $.Template.BasePath "/secret-kube-apiserver-server-tls.yaml") . | sha256sum }} - checksum/secret-kube-apiserver-static-token: {{ include (print $.Template.BasePath "/secret-kube-apiserver-static-token.yaml") . | sha256sum }} - checksum/secret-kube-controller-manager-client: {{ include (print $.Template.BasePath "/secret-kube-controller-manager-tls.yaml") . | sha256sum }} - checksum/secret-service-account-key: {{ include (print $.Template.BasePath "/secret-service-account-key.yaml") . | sha256sum }} - {{- if .Values.tls.oidc.ca }} - checksum/secret-oidc-ca: {{ include (print $.Template.BasePath "/secret-oidc-ca.yaml") . | sha256sum }} - {{- end }} - labels: - app: garden - component: kube-apiserver - {{- if .Values.networkPolicies }} - networking.gardener.cloud/to-dns: allowed - networking.gardener.cloud/to-etcd: allowed - networking.gardener.cloud/to-gardener-apiserver: allowed - networking.gardener.cloud/to-gardener-controller-manager: allowed # needed for webhooks - networking.gardener.cloud/to-identity: allowed - networking.gardener.cloud/to-ingress: allowed # needed for communication to identity - networking.gardener.cloud/to-terminal-controller-manager: allowed # needed for webhooks - networking.gardener.cloud/to-world: allowed # necessary, except for GCP because GCP puts IP table rules on the nodes that allow in-cluster routing - {{- end }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - garden - - key: component - operator: In - values: - - kube-apiserver - tolerations: - - effect: NoExecute - operator: Exists - containers: - - name: kube-apiserver - image: {{ index .Values.images "apiserver" }} - imagePullPolicy: IfNotPresent - command: - - kube-apiserver - - --enable-admission-plugins=Priority,NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook - - --disable-admission-plugins=PersistentVolumeLabel - - --allow-privileged=true - - --anonymous-auth=false - - --authorization-mode=Node,RBAC - - --token-auth-file=/srv/kubernetes/token/static_tokens.csv - - --client-ca-file=/srv/kubernetes/ca/ca.crt - - --enable-aggregator-routing=true - - --enable-bootstrap-token-auth=true - - --etcd-cafile=/srv/kubernetes/etcd/ca/ca.crt - - --etcd-certfile=/srv/kubernetes/etcd/client/tls.crt - - --etcd-keyfile=/srv/kubernetes/etcd/client/tls.key - - --etcd-servers={{ .Values.etcd.main.endpoints }} - - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP -{{ if and .Values.oidc.issuerURL .Values.oidc.clientID }} - - --oidc-issuer-url={{ .Values.oidc.issuerURL }} - - --oidc-client-id={{ .Values.oidc.clientID }} - {{- if .Values.oidc.usernameClaim }} - - --oidc-username-claim={{ .Values.oidc.usernameClaim }} - {{- end }} - {{- if .Values.oidc.usernamePrefix }} - - --oidc-username-prefix={{ .Values.oidc.usernamePrefix }} - {{- end}} - {{- if .Values.oidc.groupsClaim }} - - --oidc-groups-claim={{ .Values.oidc.groupsClaim }} - {{- end }} - {{- if .Values.oidc.groupsPrefix }} - - --oidc-groups-prefix={{ .Values.oidc.groupsPrefix }} - {{- end }} - {{- if .Values.tls.oidc.ca }} - - --oidc-ca-file=/srv/kubernetes/oidc-ca/ca.crt - {{- end }} -{{ end }} - - --profiling=false - - --proxy-client-cert-file=/srv/kubernetes/aggregator/tls.crt - - --proxy-client-key-file=/srv/kubernetes/aggregator/tls.key - - --requestheader-client-ca-file=/srv/kubernetes/aggregator-ca/ca.crt - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --requestheader-group-headers=X-Remote-Group - - --requestheader-username-headers=X-Remote-User - - --secure-port=443 - - --service-cluster-ip-range=100.64.0.0/13 - - --service-account-issuer=https://kubernetes.default.svc - - --service-account-signing-key-file=/srv/kubernetes/service-account-key/service_account.key - - --service-account-key-file=/srv/kubernetes/service-account-key/service_account.key - - --tls-cert-file=/srv/kubernetes/apiserver/tls.crt - - --tls-private-key-file=/srv/kubernetes/apiserver/tls.key - # - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA - - --v=2 - livenessProbe: - httpGet: - scheme: HTTPS - path: /healthz - port: 443 - httpHeaders: - - name: Authorization - value: Bearer {{ .Values.tls.kubeAPIServer.staticTokens.healthCheck }} - successThreshold: 1 - failureThreshold: 3 - initialDelaySeconds: 15 - periodSeconds: 30 - timeoutSeconds: 15 - readinessProbe: - httpGet: - scheme: HTTPS - path: /healthz - port: 443 - httpHeaders: - - name: Authorization - value: Bearer {{ .Values.tls.kubeAPIServer.staticTokens.healthCheck }} - successThreshold: 1 - failureThreshold: 3 - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 15 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: https - containerPort: 443 - protocol: TCP - resources: - limits: - cpu: 1200m - memory: 2000Mi - requests: - cpu: 200m - memory: 512Mi - volumeMounts: - - name: ca-kube-apiserver - mountPath: /srv/kubernetes/ca - - name: ca-etcd - mountPath: /srv/kubernetes/etcd/ca - - name: ca-front-proxy - mountPath: /srv/kubernetes/aggregator-ca - - name: etcd-client-tls - mountPath: /srv/kubernetes/etcd/client - - name: kube-apiserver - mountPath: /srv/kubernetes/apiserver - - name: service-account-key - mountPath: /srv/kubernetes/service-account-key - - name: kube-apiserver-static-token - mountPath: /srv/kubernetes/token - - name: kube-aggregator - mountPath: /srv/kubernetes/aggregator - {{- if .Values.tls.oidc.ca }} - - name: ca-oidc - mountPath: /srv/kubernetes/oidc-ca - {{- end }} - - name: kube-controller-manager - image: {{ index .Values.images "controllermanager" }} - imagePullPolicy: IfNotPresent - command: - - kube-controller-manager - - --cluster-signing-cert-file=/srv/kubernetes/ca/ca.crt - - --cluster-signing-key-file=/srv/kubernetes/ca/ca.key - - --controllers=namespace,serviceaccount,serviceaccount-token,clusterrole-aggregation,garbagecollector,csrapproving,csrcleaner,csrsigning,bootstrapsigner,tokencleaner - - --authorization-kubeconfig=/srv/kubernetes/controller-manager/kubeconfig - - --authentication-kubeconfig=/srv/kubernetes/controller-manager/kubeconfig - - --kubeconfig=/srv/kubernetes/controller-manager/kubeconfig - - --root-ca-file=/srv/kubernetes/ca/ca.crt - - --service-account-private-key-file=/srv/kubernetes/service-account-key/service_account.key - - --use-service-account-credentials=true - - --v=2 - livenessProbe: - failureThreshold: 2 - httpGet: - path: /healthz - port: 10257 - scheme: HTTPS - initialDelaySeconds: 15 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 200m - memory: 128Mi - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - name: ca-kube-apiserver - mountPath: /srv/kubernetes/ca - - name: kube-controller-manager - mountPath: /srv/kubernetes/controller-manager - - name: service-account-key - mountPath: /srv/kubernetes/service-account-key - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - terminationGracePeriodSeconds: 30 - volumes: - - name: ca-kube-apiserver - secret: - secretName: garden-kube-apiserver-ca - {{- if .Values.tls.oidc.ca }} - - name: ca-oidc - secret: - secretName: oidc-ca - {{- end }} - - name: ca-etcd - secret: - secretName: {{ .Values.etcd.secretNames.ca | default "garden-etcd-ca" }} - - name: ca-front-proxy - secret: - secretName: garden-kube-aggregator-ca - - name: kube-apiserver - secret: - secretName: garden-kube-apiserver - - name: kube-controller-manager - secret: - secretName: garden-kube-controller-manager - - name: etcd-client-tls - secret: - secretName: {{ .Values.etcd.secretNames.client | default "garden-etcd-client" }} - - name: kube-apiserver-static-token - secret: - secretName: garden-kube-apiserver-static-token - - name: service-account-key - secret: - secretName: garden-service-account-key - - name: kube-aggregator - secret: - secretName: garden-kube-aggregator diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml deleted file mode 100644 index 8fd1d924..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: garden-kube-apiserver - namespace: {{ .Release.Namespace }} -spec: - minAvailable: 2 - selector: - matchLabels: - app: garden - component: kube-apiserver diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml deleted file mode 100644 index 34701445..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-kube-aggregator-ca - namespace: {{ .Release.Namespace }} -type: Opaque -data: - ca.crt: {{ .Values.tls.kubeAggregator.ca.crt | b64enc }} - ca.key: {{ .Values.tls.kubeAggregator.ca.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml deleted file mode 100644 index 9966853f..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-kube-aggregator - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - tls.crt: {{ .Values.tls.kubeAggregator.client.crt | b64enc }} - tls.key: {{ .Values.tls.kubeAggregator.client.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml deleted file mode 100644 index 3dae3fcb..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-kube-apiserver-ca - namespace: {{ .Release.Namespace }} -type: Opaque -data: - ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - ca.key: {{ .Values.tls.kubeAPIServer.ca.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml deleted file mode 100644 index 0d6a31af..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-kube-apiserver - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - tls.crt: {{ .Values.tls.kubeAPIServer.server.crt | b64enc }} - tls.key: {{ .Values.tls.kubeAPIServer.server.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml deleted file mode 100644 index 274a90fc..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: garden-kube-apiserver-static-token - namespace: {{ .Release.Namespace }} -type: Opaque -data: - static_tokens.csv: {{ printf "%s,kube-apiserver-health-check,kube-apiserver-health-check," .Values.tls.kubeAPIServer.staticTokens.healthCheck | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml deleted file mode 100644 index 4abad406..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-kube-controller-manager - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - tls.crt: {{ .Values.tls.kubeControllerManager.crt | b64enc }} - tls.key: {{ .Values.tls.kubeControllerManager.key | b64enc }} - kubeconfig: {{ include "garden.kubeconfig-controller-manager" . | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml deleted file mode 100644 index 6a9d84d5..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-kubeconfig-for-admin - namespace: {{ .Release.Namespace }} -type: Opaque -data: - ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} - tls.crt: {{ .Values.tls.admin.crt | b64enc }} - tls.key: {{ .Values.tls.admin.key | b64enc }} - kubeconfig: {{ include "garden.kubeconfig-admin" . | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-oidc-ca.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-oidc-ca.yaml deleted file mode 100644 index 28f67d04..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-oidc-ca.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.tls.oidc.ca }} -apiVersion: v1 -kind: Secret -metadata: - name: oidc-ca - namespace: {{ .Release.Namespace }} -type: Opaque -data: - ca.crt: {{ .Values.tls.oidc.ca | b64enc }} -{{- end }} \ No newline at end of file diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml deleted file mode 100644 index b089acaf..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Secret -metadata: - name: garden-service-account-key - namespace: {{ .Release.Namespace }} -type: Opaque -data: - service_account.key: {{ .Values.tls.serviceAccountKey | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml deleted file mode 100644 index ef92f455..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - name: apiserver-ingress - namespace: {{ .Release.Namespace }} -spec: - ingressClassName: nginx - rules: - - host: {{ .Values.apiServer.hostname }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ .Values.serviceName }} - port: - number: 443 - tls: - - hosts: - - {{ .Values.apiServer.hostname }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml deleted file mode 100644 index f6f15d36..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.serviceName }} - namespace: {{ .Release.Namespace }} - labels: - app: garden - component: kube-apiserver -spec: - type: ClusterIP - selector: - app: garden - component: kube-apiserver - ports: - - name: kube-apiserver - protocol: TCP - port: 443 - targetPort: 443 diff --git a/control-plane/roles/gardener/files/kube-apiserver/values.yaml b/control-plane/roles/gardener/files/kube-apiserver/values.yaml deleted file mode 100644 index 113c05c8..00000000 --- a/control-plane/roles/gardener/files/kube-apiserver/values.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -namespace: xxx -serviceName: garden-kube-apiserver -images: - apiserver: k8s.gcr.io/kube-apiserver:v1.19.16 - controllermanager: k8s.gcr.io/kube-controller-manager:v1.19.16 - -replicas: 3 -apiServer: - hostname: 127.0.0.1 - port: 443 - serviceName: garden-kube-apiserver - -oidc: - issuerURL: - clientID: - usernameClaim: - usernamePrefix: - groupsClaim: - groupsPrefix: - -tls: - kubeAPIServer: - ca: - crt: ca-certificate - key: ca-key - server: - crt: server-certificate - key: server-key - staticTokens: - healthCheck: token - kubeControllerManager: - crt: client-certificate - key: client-key - admin: - crt: client-certificate - key: client-key - serviceAccountKey: key - oidc: - ca: diff --git a/control-plane/roles/gardener/tasks/gardener_control_plane_obsolete.yaml b/control-plane/roles/gardener/tasks/gardener_control_plane_obsolete.yaml deleted file mode 100644 index 67de7d3f..00000000 --- a/control-plane/roles/gardener/tasks/gardener_control_plane_obsolete.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: Deploy Gardener Control Plane (in virtual apiserver) - include_role: - name: ansible-common/roles/helm-chart - vars: - helm_timeout: "600s" - helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/controlplane/charts/application" - helm_release_name: controlplane - helm_target_namespace: garden - helm_value_file_template: gardener-control-plane-values.j2 - helm_kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" - -- name: Deploy Gardener Control Plane - include_role: - name: ansible-common/roles/helm-chart - vars: - helm_timeout: "600s" - helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/controlplane/charts/runtime" - helm_release_name: controlplane - helm_target_namespace: garden - helm_value_file_template: gardener-control-plane-values.j2 - -- name: Wait until gardener runtime components are ready - command: echo - changed_when: false - retries: 10 - delay: 6 - until: - - lookup('k8s', api_version='apps/v1', kind='Deployment', namespace='garden', resource_name='gardener-apiserver').get('status', {}).get('readyReplicas', 0) >= 1 - - lookup('k8s', api_version='apps/v1', kind='Deployment', namespace='garden', resource_name='garden-kube-apiserver').get('status', {}).get('readyReplicas', 0) >= 3 - - lookup('k8s', api_version='apps/v1', kind='Deployment', namespace='garden', resource_name='gardener-controller-manager').get('status', {}).get('readyReplicas', 0) >= 1 diff --git a/control-plane/roles/gardener/tasks/gardener_operator.yaml b/control-plane/roles/gardener/tasks/gardener_operator.yaml deleted file mode 100644 index bed25eff..00000000 --- a/control-plane/roles/gardener/tasks/gardener_operator.yaml +++ /dev/null @@ -1,143 +0,0 @@ ---- -- name: Create backup infrastructure secret - k8s: - definition: - apiVersion: v1 - kind: Secret - metadata: - name: virtual-garden-etcd-main-backup-secret - namespace: garden - type: Opaque - data: "{{ gardener_backup_infrastructure_secret }}" - apply: yes - -# TODO: prepare migration here -# - label existing secrets like CA, ETCD encryption key, accordingly -# - scale down existing components (gardener control plane + virtual garden with ETCD) -# -# https://github.com/gardener/gardener/blob/v1.100.2/docs/concepts/operator.md#migrating-an-existing-gardener-landscape-to-gardener-operator - -- name: Deploy Gardener Operator - include_role: - name: ansible-common/roles/helm-chart - vars: - helm_timeout: "600s" - helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/operator" - helm_release_name: operator - helm_target_namespace: garden - helm_value_file_template: gardener-operator-values.j2 - -- name: Create Garden - k8s: - definition: "{{ lookup('template', 'garden.yaml') }}" - apply: yes - -- name: Wait until Garden is ready - kubernetes.core.k8s_info: - api_version: "operator.gardener.cloud/v1alpha1" - kind: Garden - name: "local" - wait: yes - wait_condition: - status: "True" - type: "{{ item }}" - wait_timeout: 900 - loop: - - VirtualComponentsHealthy - - RuntimeComponentsHealthy - -# TODO: we should expose through istio and Gardener Operator in the future? -- name: Create ingress for virtual kube-apiserver - k8s: - definition: - apiVersion: networking.k8s.io/v1 - kind: Ingress - metadata: - annotations: - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - name: apiserver-ingress - namespace: garden - spec: - ingressClassName: nginx - rules: - - host: "{{ gardener_virtual_api_server_public_dns }}" - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: virtual-garden-kube-apiserver - port: - number: 443 - tls: - - hosts: - - "{{ gardener_virtual_api_server_public_dns }}" - apply: yes - -- name: Create virtual garden access secret - k8s: - apply: yes - definition: - apiVersion: v1 - kind: Secret - type: Opaque - metadata: - name: shoot-access-virtual-garden - namespace: garden - labels: - resources.gardener.cloud/purpose: token-requestor - resources.gardener.cloud/class: shoot - annotations: - serviceaccount.resources.gardener.cloud/name: virtual-garden-user - serviceaccount.resources.gardener.cloud/namespace: kube-system - serviceaccount.resources.gardener.cloud/token-expiration-duration: 3h - -- name: Create virtual garden access managed resource secret - k8s: - apply: yes - definition: - apiVersion: v1 - kind: Secret - metadata: - name: managedresource-virtual-garden-access - namespace: garden - type: Opaque - stringData: - clusterrolebinding____gardener.cloud.virtual-garden-access.yaml: | - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: gardener.cloud.sap:virtual-garden - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - subjects: - - kind: ServiceAccount - name: virtual-garden-user - namespace: kube-system - -- name: Create virtual garden access managed resource - k8s: - apply: yes - definition: - apiVersion: resources.gardener.cloud/v1alpha1 - kind: ManagedResource - metadata: - name: virtual-garden-access - namespace: garden - spec: - secretRefs: - - name: managedresource-virtual-garden-access - -- name: Create kubeconfig for accessing the virtual garden - copy: - dest: "{{ gardener_kube_apiserver_kubeconfig_path }}" - content: "{{ (gardener_virtual_api_server_public_dns + ':' + gardener_virtual_api_server_public_wait_port | string) | kubeconfig_from_token(gardener_kube_api_server_ca, lookup('k8s', api_version='v1', kind='Secret', namespace='garden', resource_name='shoot-access-virtual-garden').get('data', {}).get('token') | b64decode, prepend_https=true) }}" - -- name: Wait for garden-kube-apiserver - wait_for: - host: "{{ gardener_virtual_api_server_public_dns }}" - port: "{{ gardener_virtual_api_server_public_wait_port }}" - timeout: 60 diff --git a/control-plane/roles/gardener/tasks/main.yaml b/control-plane/roles/gardener/tasks/main.yaml index 73a2e393..e7a87388 100644 --- a/control-plane/roles/gardener/tasks/main.yaml +++ b/control-plane/roles/gardener/tasks/main.yaml @@ -11,72 +11,18 @@ - gardener_repo_ref is defined - gardener_gardenlet_image_tag is defined - gardener_gardenlet_image_name is defined - - gardener_etcd_repo_ref is defined - - gardener_apiserver_image_tag is defined - - gardener_controller_manager_image_tag is defined - - gardener_scheduler_image_tag is defined - gardener_extension_provider_metal_image_tag is defined - gardener_os_controller_image_tag is defined - metal_cloud_controller_manager_image_tag is defined - gardener_networking_calico_image_tag is defined - csi_lvm_controller_image_tag is defined - csi_lvm_provisioner_image_tag is defined - - gardener_api_server_ca is not none - - gardener_api_server_ca_key is not none - - gardener_api_server_cert is not none - - gardener_api_server_key is not none - - gardener_api_server_client_cert is not none - - gardener_api_server_client_key is not none - - gardener_kube_aggregator_client_cert is not none - - gardener_kube_aggregator_client_key is not none - - gardener_kube_controller_manager_client_cert is not none - - gardener_kube_controller_manager_client_key is not none - - gardener_admin_client_cert is not none - - gardener_admin_client_key is not none - - gardener_service_account_client_key is not none - - gardener_admission_controller_ca is not none - - gardener_admission_controller_cert is not none - - gardener_admission_controller_key is not none - - gardener_controller_manager_ca is not none - - gardener_controller_manager_cert is not none - - gardener_controller_manager_key is not none - - gardener_admission_metal_ca is not none - - gardener_admission_metal_cert is not none - - gardener_admission_metal_key is not none - - gardener_etcd_ca_cert is not none - - gardener_etcd_cert is not none - - gardener_etcd_cert_key is not none - - gardener_etcd_client_cert is not none - - gardener_etcd_client_key is not none - - gardener_virtual_api_server_public_dns is not none - - gardener_virtual_api_server_healthcheck_static_token is not none - gardener_dns_domain is not none - gardener_dns_provider is not none - gardener_cloud_profile_metal_api_url is not none - gardener_cloud_profile_metal_api_hmac is not none - - gardener_backup_infrastructure is not none and gardener_backup_infrastructure.provider in ["local", "gcp", "S3"] - gardener_cert_management_issuer_email is not none -- name: Deploy required Seed CRDs - k8s: - definition: "{{ item.definition }}" - apply: yes - loop: - - name: hvpas - definition: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener/' + gardener_repo_ref + '/example/seed-crds/10-crd-autoscaling.k8s.io_hvpas.yaml', split_lines=False) | from_yaml_all | list)[0] }}" - - name: vpas - definition: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener/' + gardener_repo_ref + '/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml', split_lines=False) | from_yaml_all | list)[0] }}" - loop_control: - label: "{{ item.name }}" - -- name: Create garden namespace - k8s: - definition: - apiVersion: v1 - kind: Namespace - metadata: - name: garden - - name: Fail when provider unsupported fail: msg: "provider is unsupported: {{ metal_control_plane_host_provider }}" @@ -93,36 +39,6 @@ when: - metal_control_plane_host_provider == "metal" -- name: Create backup directory for local deployment - import_tasks: local_backup.yaml - when: gardener_backup_infrastructure.provider == "local" - -- name: Clone Gardener - git: - repo: "{{ gardener_repo_url }}" - dest: "{{ gardener_local_tmp_dir }}/gardener" - depth: 1 - version: "{{ gardener_repo_ref }}" - -- name: Deploy Gardener Operator - import_tasks: gardener_operator.yaml - when: gardener_operator_enabled - -- name: Deploy virtual garden (old way without operator) - import_tasks: virtual_garden.yaml - when: not gardener_operator_enabled - -- name: Create garden namespace (in virtual apiserver) - k8s: - definition: - apiVersion: v1 - kind: Namespace - metadata: - name: garden - labels: - app: gardener - kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" - - name: Deploy domain secrets (in virtual apiserver) k8s: definition: @@ -146,10 +62,6 @@ - internal-domain - default-domain -- name: Deploy Gardener Control Plane (old way without operator) - import_tasks: gardener_control_plane_obsolete.yaml - when: not gardener_operator_enabled - - name: Register admission controllers import_tasks: admission_controllers.yaml diff --git a/control-plane/roles/gardener/tasks/seed.yaml b/control-plane/roles/gardener/tasks/seed.yaml index f4ccb8a9..1c294197 100644 --- a/control-plane/roles/gardener/tasks/seed.yaml +++ b/control-plane/roles/gardener/tasks/seed.yaml @@ -11,7 +11,6 @@ data: "{{ gardener_backup_infrastructure_secret }}" kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" apply: yes - when: gardener_backup_infrastructure_secret - name: Add service account for gardener seeds k8s: diff --git a/control-plane/roles/gardener/tasks/virtual_garden.yaml b/control-plane/roles/gardener/tasks/virtual_garden.yaml deleted file mode 100644 index 1761603d..00000000 --- a/control-plane/roles/gardener/tasks/virtual_garden.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -- name: Clone Gardener etcd (with backup restore) - git: - repo: "{{ gardener_etcd_repo_url }}" - dest: "{{ gardener_local_tmp_dir }}/etcd-backup-restore" - depth: 1 - version: "{{ gardener_etcd_repo_ref }}" - -- name: Deploy Gardener etcd (with backup restore) - include_role: - name: ansible-common/roles/helm-chart - vars: - helm_timeout: "600s" - helm_chart: "{{ gardener_local_tmp_dir }}/etcd-backup-restore/chart/etcd-backup-restore" - helm_release_name: etcd-{{ metal_control_plane_stage_name }} - helm_target_namespace: garden - helm_value_file_template: etcd-values.j2 - -# The virtual garden setup requires one service through which the soil and the garden-apiserver can communicate -# This needs to have the same, static ip address, which needs to be chosen randomly from the service network -- name: Determine current gardener-apiserver service cluster ip - set_fact: - _gardener_api_server_current_service_ip: "{{ (lookup('k8s', api_version='v1', kind='Service', namespace='garden', resource_name='gardener-apiserver') | default({}, true)).get('spec', {}).get('clusterIP', none) }}" - -- name: Calculate gardener-kube-apiserver service cluster IP - set_fact: - gardener_virtual_api_server_svc_cluster_ip: "{{ _gardener_api_server_current_service_ip if _gardener_api_server_current_service_ip else _gardener_soil_service_cidr | network_cidr_add(add=gardener_virtual_api_server_svc_cluster_ip_add) }}" - -- name: Deploy kube-apiserver for virtual garden - include_role: - name: ansible-common/roles/helm-chart - vars: - helm_timeout: "600s" - helm_chart_custom_folder: "{{ ansible_parent_role_paths[0] }}/files/kube-apiserver" - helm_chart: "./kube-apiserver" - helm_release_name: virtual-garden - helm_target_namespace: garden - helm_value_file_template: kube-apiserver-values.j2 - helm_chart_inject_config_hash: yes - -- name: Read admin kubeconfig for garden-kube-apiserver - copy: - dest: "{{ gardener_kube_apiserver_kubeconfig_path }}" - content: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace='garden', resource_name='garden-kubeconfig-for-admin').get('data', {}).get('kubeconfig') | b64decode }}" - -- name: Wait for garden-kube-apiserver - wait_for: - host: "{{ gardener_virtual_api_server_public_dns }}" - port: "{{ gardener_virtual_api_server_public_wait_port }}" - timeout: 60 diff --git a/control-plane/roles/gardener/templates/etcd-values.j2 b/control-plane/roles/gardener/templates/etcd-values.j2 deleted file mode 100644 index 59639fb8..00000000 --- a/control-plane/roles/gardener/templates/etcd-values.j2 +++ /dev/null @@ -1,34 +0,0 @@ -images: - etcdBackupRestore: - tag: "{{ gardener_etcd_repo_ref }}" - -{% if gardener_backup_infrastructure_secret %} -backup: - storageContainer: {{ gardener_backup_infrastructure.bucket }} -{% if gardener_backup_infrastructure.provider == "gcp" %} - storageProvider: "GCS" - gcs: - serviceAccountJson: {{ gardener_backup_infrastructure_service_account_json | to_json }} -{% elif gardener_backup_infrastructure.provider == "S3" %} - storageProvider: "ECS" - ecs: - endpoint: "{{ gardener_backup_infrastructure_secret.endpoint | b64decode }}" - accessKeyID: "{{ gardener_backup_infrastructure_secret.accessKeyID | b64decode }}" - secretAccessKey: "{{ gardener_backup_infrastructure_secret.secretAccessKey | b64decode}}" -{% endif %} -{% endif %} - - schedule: {{ gardener_etcd_backup_schedule }} - deltaSnapshotPeriod: {{ gardener_etcd_snapshot_period }} - garbageCollectionPeriod: {{ gardener_etcd_garbage_collection_period }} - -etcdTLS: - caBundle: | - {{ gardener_etcd_ca_cert | indent(width=4, first=false) }} - crt: | - {{ gardener_etcd_cert | indent(width=4, first=false) }} - key: | - {{ gardener_etcd_cert_key | indent(width=4, first=false) }} - -resources: - etcd: {{ gardener_etcd_resources | to_json }} diff --git a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 deleted file mode 100644 index b9dec2c0..00000000 --- a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 +++ /dev/null @@ -1,95 +0,0 @@ ---- -global: - admission: - image: - repository: {{ gardener_admission_controller_image_name }} - tag: {{ gardener_admission_controller_image_tag }} - enabled: true - kubeconfig: | - {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} - config: - server: - webhooks: - tls: - caBundle: | - {{ gardener_admission_controller_ca | indent(width=14, first=false) }} - crt: | - {{ gardener_admission_controller_cert | indent(width=14, first=false) }} - key: | - {{ gardener_admission_controller_key | indent(width=14, first=false) }} - - apiserver: - replicaCount: {{ gardener_apiserver_replicas }} - clusterIdentity: gardener-soil-{{ metal_control_plane_stage_name }} - resources: {{ gardener_apiserver_resources | to_json }} - featureGates: {{ gardener_apiserver_feature_gates | to_json }} - image: - repository: {{ gardener_apiserver_image_name }} - tag: {{ gardener_apiserver_image_tag }} - etcd: - useSidecar: false - servers: https://etcd-{{ metal_control_plane_stage_name }}-etcd-client:2379 - caBundle: | - {{ gardener_etcd_ca_cert | indent(width=8, first=false) }} - tls: - crt: | - {{ gardener_etcd_client_cert | indent(width=10, first=false) }} - key: | - {{ gardener_etcd_client_key | indent(width=10, first=false) }} - insecureSkipTLSVerify: false - caBundle: | - {{ gardener_api_server_ca | indent(width=6, first=false) }} - tls: - crt: | - {{ gardener_api_server_cert | indent(width=8, first=false) }} - key: | - {{ gardener_api_server_key | indent(width=8, first=false) }} - kubeconfig: | - {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} - - shootAdminKubeconfigMaxExpiration: {{ gardener_apiserver_shoot_kubeconfig_max_expiration }} - shootViewerKubeconfigMaxExpiration: {{ gardener_apiserver_shoot_kubeconfig_max_expiration }} - - vpa: {{ gardener_apiserver_vpa }} - hvpa: - enabled: true - - scheduler: - resources: {{ gardener_scheduler_resources | to_json }} - image: - repository: {{ gardener_scheduler_image_name }} - tag: {{ gardener_scheduler_image_tag }} - kubeconfig: | - {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} - - controller: - resources: {{ gardener_controller_manager_resources | to_json }} - image: - repository: {{ gardener_controller_manager_image_name }} - tag: {{ gardener_controller_manager_image_tag }} - - config: - featureGates: - # ssh key pair rotation is removed in 1.51, but was enabled by default in 1.45 - # TODO: comment in this line prior to deploying g/g 1.45 - # RotateSSHKeypairOnMaintenance: false - server: - https: - tls: - caBundle: | - {{ gardener_controller_manager_ca | indent(width=14, first=false) }} - crt: | - {{ gardener_controller_manager_cert | indent(width=14, first=false) }} - key: | - {{ gardener_controller_manager_key | indent(width=14, first=false) }} - controllers: - shootMaintenance: - enableShootCoreAddonRestarter: false - kubeconfig: | - {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} - - deployment: - virtualGarden: - enabled: true - clusterIP: "{{ gardener_virtual_api_server_svc_cluster_ip }}" - createNamespace: false diff --git a/control-plane/roles/gardener/templates/kube-apiserver-values.j2 b/control-plane/roles/gardener/templates/kube-apiserver-values.j2 deleted file mode 100644 index b15dd610..00000000 --- a/control-plane/roles/gardener/templates/kube-apiserver-values.j2 +++ /dev/null @@ -1,79 +0,0 @@ ---- -images: - apiserver: {{ gardener_virtual_api_server_image_name }}:{{ gardener_virtual_api_server_image_tag }} - controllermanager: {{ gardener_virtual_controller_manager_image_name }}:{{ gardener_virtual_controller_manager_image_tag }} - -apiServer: - hostname: {{ gardener_virtual_api_server_public_dns }} - port: {{ gardener_virtual_api_server_port }} - serviceName: garden-kube-apiserver - -oidc: - issuerURL: {% if gardener_virtual_api_oidc_issuer_url %}{{ gardener_virtual_api_oidc_issuer_url }}{% endif %} - - clientID: {% if gardener_virtual_api_oidc_client_id %}{{ gardener_virtual_api_oidc_client_id }}{% endif %} - -{% if gardener_virtual_api_oidc_username_claim %} - usernameClaim: {{ gardener_virtual_api_oidc_username_claim }} -{% endif %} -{% if gardener_virtual_api_oidc_username_prefix %} - usernamePrefix: "{{ gardener_virtual_api_oidc_username_prefix }}" -{% endif %} -{% if gardener_virtual_api_oidc_groups_claim %} - groupsClaim: {{ gardener_virtual_api_oidc_groups_claim }} -{% endif %} -{% if gardener_virtual_api_oidc_groups_prefix %} - groupsPrefix: "{{ gardener_virtual_api_oidc_groups_prefix }}" -{% endif %} - -tls: - kubeAPIServer: - ca: - crt: | - {{ gardener_kube_api_server_ca | indent(width=8, first=false) }} - key: | - {{ gardener_kube_api_server_ca_key | indent(width=8, first=false) }} - server: - crt: | - {{ gardener_kube_api_server_cert | indent(width=8, first=false) }} - key: | - {{ gardener_kube_api_server_key | indent(width=8, first=false) }} - staticTokens: - healthCheck: {{ gardener_virtual_api_server_healthcheck_static_token }} - kubeAggregator: - ca: - crt: | - {{ gardener_kube_api_server_ca | indent(width=8, first=false) }} - key: | - {{ gardener_kube_api_server_ca_key | indent(width=8, first=false) }} - client: - crt: | - {{ gardener_kube_aggregator_client_cert | indent(width=8, first=false) }} - key: | - {{ gardener_kube_aggregator_client_key | indent(width=8, first=false) }} - kubeControllerManager: - crt: | - {{ gardener_kube_controller_manager_client_cert | indent(width=6, first=false) }} - key: | - {{ gardener_kube_controller_manager_client_key | indent(width=6, first=false) }} - admin: - crt: | - {{ gardener_admin_client_cert | indent(width=6, first=false) }} - key: | - {{ gardener_admin_client_key | indent(width=6, first=false) }} - serviceAccountKey: | - {{ gardener_service_account_client_key | indent(width=4, first=false) }} -{% if gardener_virtual_api_oidc_ca %} - oidc: - ca: | - {{ gardener_virtual_api_oidc_ca | indent(width=6, first=false) }} -{% endif %} - -etcd: - main: - endpoints: https://etcd-{{ metal_control_plane_stage_name }}-etcd-client:2379 - secretNames: - ca: etcd-{{ metal_control_plane_stage_name }}-etcd-ca - client: etcd-{{ metal_control_plane_stage_name }}-etcd-tls - -networkPolicies: false