Skip to content

Commit

Permalink
refactor spark-k8s-operator for helm-addon (#309)
Browse files Browse the repository at this point in the history
* refactor spark-k8s-operator for helm-addon

* terraform-docs: automated action

* fix var name

* fix copy pasta

* revert module name

* terraform-docs: automated action

* account for spark and spark operator irsa policies

* terraform-docs: automated action

* add module

* irsa config is not required

* terraform-docs: automated action

* fix variable names

* remove comment

* terraform-docs: automated action

* spark operator addon update

* terraform-docs: automated action

* Addon context added to spark operator

* precommit format update

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Vara Bonthu <vara.bonthu@gmail.com>
  • Loading branch information
3 people authored Mar 9, 2022
1 parent 4388556 commit ea0922f
Show file tree
Hide file tree
Showing 12 changed files with 97 additions and 173 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# NOTE: batchScheduler and Webhook should be enabled to leverage batch schedulers like Volcano or YuniKorn
# Default values for spark-operator.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# replicaCount -- Desired number of pods, leaderElection will be enabled
# if this is greater than 1
Expand All @@ -10,7 +12,7 @@ image:
# -- Image pull policy
pullPolicy: IfNotPresent
# -- if set, override the image tag whose default is the chart appVersion.
tag: "v1beta2-1.3.3-3.1.1"
tag: ""

# -- Image pull secrets
imagePullSecrets: []
Expand All @@ -35,11 +37,15 @@ serviceAccounts:
create: true
# -- Optional name for the spark service account
name: ""
# -- Optional annotations for the spark service account
annotations: {}
sparkoperator:
# -- Create a service account for the operator
create: true
# -- Optional name for the operator service account
name: ""
# -- Optional annotations for the operator service account
annotations: {}

# -- Set this if running spark jobs in a different namespace than the operator
sparkJobNamespace: ""
Expand Down Expand Up @@ -68,9 +74,15 @@ podSecurityContext: {}
# securityContext -- Operator container security context
securityContext: {}

# volumes - Operator volumes
volumes: []

# volumeMounts - Operator volumeMounts
volumeMounts: []

webhook:
# -- Enable webhook server
enable: true
enable: false
# -- Webhook service port
port: 8080
# -- The webhook server will only operate on namespaces with this label, specified in the form key1=value1,key2=value2.
Expand Down Expand Up @@ -141,7 +153,7 @@ resources: {}

batchScheduler:
# -- Enable batch scheduler for spark jobs scheduling. If enabled, users can specify batch scheduler name in spark application
enable: true
enable: false

resourceQuotaEnforcement:
# -- Whether to enable the ResourceQuota enforcement for SparkApplication resources.
Expand Down
4 changes: 2 additions & 2 deletions examples/analytics/spark-k8s-operator/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ module "kubernetes-addons" {
name = "spark-operator"
chart = "spark-operator"
repository = "https://googlecloudplatform.github.io/spark-on-k8s-operator"
version = "1.1.15"
namespace = "spark-k8s-operator"
version = "1.1.19"
namespace = "spark-operator"
timeout = "300"
create_namespace = true
values = [templatefile("${path.module}/helm_values/spark-k8s-operator-values.yaml", {})]
Expand Down
4 changes: 4 additions & 0 deletions modules/kubernetes-addons/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
| <a name="input_node_termination_handler_irsa_permissions_boundary"></a> [node\_termination\_handler\_irsa\_permissions\_boundary](#input\_node\_termination\_handler\_irsa\_permissions\_boundary) | IAM Policy ARN for IRSA IAM role permissions boundary | `string` | `""` | no |
| <a name="input_node_termination_handler_irsa_policies"></a> [node\_termination\_handler\_irsa\_policies](#input\_node\_termination\_handler\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
| <a name="input_prometheus_helm_config"></a> [prometheus\_helm\_config](#input\_prometheus\_helm\_config) | Community Prometheus Helm Chart config | `any` | `{}` | no |
| <a name="input_spark_irsa_permissions_boundary"></a> [spark\_irsa\_permissions\_boundary](#input\_spark\_irsa\_permissions\_boundary) | IAM Policy ARN for IRSA IAM role permissions boundary for Spark App | `string` | `""` | no |
| <a name="input_spark_irsa_policies"></a> [spark\_irsa\_policies](#input\_spark\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts for Spark App | `list(string)` | `[]` | no |
| <a name="input_spark_k8s_operator_helm_config"></a> [spark\_k8s\_operator\_helm\_config](#input\_spark\_k8s\_operator\_helm\_config) | Spark on K8s Operator Helm Chart config | `any` | `{}` | no |
| <a name="input_spark_operator_irsa_permissions_boundary"></a> [spark\_operator\_irsa\_permissions\_boundary](#input\_spark\_operator\_irsa\_permissions\_boundary) | IAM Policy ARN for IRSA IAM role permissions boundary for Spark Operator | `string` | `""` | no |
| <a name="input_spark_operator_irsa_policies"></a> [spark\_operator\_irsa\_policies](#input\_spark\_operator\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts for Spark Operator | `list(string)` | `[]` | no |
| <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no |
| <a name="input_traefik_helm_config"></a> [traefik\_helm\_config](#input\_traefik\_helm\_config) | Traefik Helm Chart config | `any` | `{}` | no |
| <a name="input_vpa_helm_config"></a> [vpa\_helm\_config](#input\_vpa\_helm\_config) | VPA Helm Chart config | `any` | `null` | no |
Expand Down
2 changes: 1 addition & 1 deletion modules/kubernetes-addons/helm-addon/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ No requirements.
|------|-------------|------|---------|:--------:|
| <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br> aws_caller_identity_account_id = string<br> aws_caller_identity_arn = string<br> aws_eks_cluster_endpoint = string<br> aws_partition_id = string<br> aws_region_name = string<br> eks_cluster_id = string<br> eks_oidc_issuer_url = string<br> eks_oidc_provider_arn = string<br> tags = map(string)<br> })</pre> | n/a | yes |
| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Add-on helm chart config, provide repository and version at the minimum.<br>See https://registry.terraform.io/providers/hashicorp/helm/latest/docs. | `any` | n/a | yes |
| <a name="input_irsa_config"></a> [irsa\_config](#input\_irsa\_config) | Input configuration for IRSA module | <pre>object({<br> kubernetes_namespace = string<br> create_kubernetes_namespace = optional(bool)<br> kubernetes_service_account = string<br> create_kubernetes_service_account = optional(bool)<br> iam_role_path = optional(string)<br> irsa_iam_policies = optional(list(string))<br> irsa_iam_permissions_boundary = optional(string)<br> })</pre> | n/a | yes |
| <a name="input_irsa_config"></a> [irsa\_config](#input\_irsa\_config) | Input configuration for IRSA module | <pre>object({<br> kubernetes_namespace = string<br> create_kubernetes_namespace = optional(bool)<br> kubernetes_service_account = string<br> create_kubernetes_service_account = optional(bool)<br> iam_role_path = optional(string)<br> irsa_iam_policies = optional(list(string))<br> irsa_iam_permissions_boundary = optional(string)<br> })</pre> | `null` | no |
| <a name="input_manage_via_gitops"></a> [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no |
| <a name="input_set_sensitive_values"></a> [set\_sensitive\_values](#input\_set\_sensitive\_values) | Forced set\_sensitive values | `any` | `[]` | no |
| <a name="input_set_values"></a> [set\_values](#input\_set\_values) | Forced set values | `any` | `[]` | no |
Expand Down
1 change: 1 addition & 0 deletions modules/kubernetes-addons/helm-addon/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ variable "irsa_config" {
irsa_iam_permissions_boundary = optional(string)
})
description = "Input configuration for IRSA module"
default = null
}

variable "addon_context" {
Expand Down
1 change: 1 addition & 0 deletions modules/kubernetes-addons/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ module "spark_k8s_operator" {
source = "./spark-k8s-operator"
helm_config = var.spark_k8s_operator_helm_config
manage_via_gitops = var.argocd_manage_add_ons
addon_context = local.addon_context
}

module "traefik" {
Expand Down
29 changes: 8 additions & 21 deletions modules/kubernetes-addons/spark-k8s-operator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,6 @@ It uses Kubernetes custom resources for specifying, running, and surfacing statu
#### AWS Service annotations for Nginx Ingress Controller

<!--- BEGIN_TF_DOCS --->
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
SPDX-License-Identifier: MIT-0

Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

## Requirements

No requirements.
Expand All @@ -30,24 +14,27 @@ No requirements.

| Name | Version |
|------|---------|
| <a name="provider_helm"></a> [helm](#provider\_helm) | n/a |
| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | n/a |

## Modules

No modules.
| Name | Source | Version |
|------|--------|---------|
| <a name="module_helm_addon"></a> [helm\_addon](#module\_helm\_addon) | ../helm-addon | n/a |

## Resources

| Name | Type |
|------|------|
| [helm_release.spark_k8s_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [kubernetes_namespace_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |

## Inputs

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Spark on K8s Operator Helm Chart Configuration | `any` | `{}` | no |
| <a name="input_manage_via_gitops"></a> [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no |
| <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br> aws_caller_identity_account_id = string<br> aws_caller_identity_arn = string<br> aws_eks_cluster_endpoint = string<br> aws_partition_id = string<br> aws_region_name = string<br> eks_cluster_id = string<br> eks_oidc_issuer_url = string<br> eks_oidc_provider_arn = string<br> tags = map(string)<br> })</pre> | n/a | yes |
| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm provider config for Spark K8s Operator | `any` | `{}` | no |
| <a name="input_manage_via_gitops"></a> [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps | `bool` | `false` | no |

## Outputs

Expand Down
46 changes: 10 additions & 36 deletions modules/kubernetes-addons/spark-k8s-operator/locals.tf
Original file line number Diff line number Diff line change
@@ -1,41 +1,15 @@

locals {
name = "spark-operator"

default_helm_config = {
name = "spark-operator"
chart = "spark-operator"
repository = "https://googlecloudplatform.github.io/spark-on-k8s-operator"
version = "1.1.19"
namespace = "spark-k8s-operator"
timeout = "1200"
create_namespace = true
values = null
set = []
set_sensitive = null
lint = false
verify = false
keyring = ""
repository_key_file = ""
repository_cert_file = ""
repository_ca_file = ""
repository_username = ""
repository_password = ""
disable_webhooks = false
reuse_values = false
reset_values = false
force_update = false
recreate_pods = false
cleanup_on_fail = false
max_history = 0
atomic = false
skip_crds = false
render_subchart_notes = true
disable_openapi_validation = false
wait = true
wait_for_jobs = false
dependency_update = false
replace = false
description = "The spark_k8s_operator HelmChart Ingress Controller deployment configuration"
postrender = ""
name = local.name
chart = local.name
repository = "https://googlecloudplatform.github.io/spark-on-k8s-operator"
version = "1.1.19"
namespace = local.name
description = "The spark_k8s_operator HelmChart Ingress Controller deployment configuration"
values = null
timeout = "1200"
}

helm_config = merge(
Expand Down
84 changes: 13 additions & 71 deletions modules/kubernetes-addons/spark-k8s-operator/main.tf
Original file line number Diff line number Diff line change
@@ -1,77 +1,19 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: MIT-0
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of this
* software and associated documentation files (the "Software"), to deal in the Software
* without restriction, including without limitation the rights to use, copy, modify,
* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
module "helm_addon" {
source = "../helm-addon"
manage_via_gitops = var.manage_via_gitops
helm_config = local.helm_config
irsa_config = null
addon_context = var.addon_context

resource "helm_release" "spark_k8s_operator" {
count = var.manage_via_gitops ? 0 : 1
name = local.helm_config["name"]
repository = local.helm_config["repository"]
chart = local.helm_config["chart"]
version = local.helm_config["version"]
namespace = local.helm_config["namespace"]
timeout = local.helm_config["timeout"]
values = local.helm_config["values"]
create_namespace = local.helm_config["create_namespace"]
lint = local.helm_config["lint"]
description = local.helm_config["description"]
repository_key_file = local.helm_config["repository_key_file"]
repository_cert_file = local.helm_config["repository_cert_file"]
repository_ca_file = local.helm_config["repository_ca_file"]
repository_username = local.helm_config["repository_username"]
repository_password = local.helm_config["repository_password"]
verify = local.helm_config["verify"]
keyring = local.helm_config["keyring"]
disable_webhooks = local.helm_config["disable_webhooks"]
reuse_values = local.helm_config["reuse_values"]
reset_values = local.helm_config["reset_values"]
force_update = local.helm_config["force_update"]
recreate_pods = local.helm_config["recreate_pods"]
cleanup_on_fail = local.helm_config["cleanup_on_fail"]
max_history = local.helm_config["max_history"]
atomic = local.helm_config["atomic"]
skip_crds = local.helm_config["skip_crds"]
render_subchart_notes = local.helm_config["render_subchart_notes"]
disable_openapi_validation = local.helm_config["disable_openapi_validation"]
wait = local.helm_config["wait"]
wait_for_jobs = local.helm_config["wait_for_jobs"]
dependency_update = local.helm_config["dependency_update"]
replace = local.helm_config["replace"]

postrender {
binary_path = local.helm_config["postrender"]
}

dynamic "set" {
iterator = each_item
for_each = local.helm_config["set"] == null ? [] : local.helm_config["set"]

content {
name = each_item.value.name
value = each_item.value.value
}
}
depends_on = [kubernetes_namespace_v1.this]
}

dynamic "set_sensitive" {
iterator = each_item
for_each = local.helm_config["set_sensitive"] == null ? [] : local.helm_config["set_sensitive"]
resource "kubernetes_namespace_v1" "this" {
metadata {
name = local.helm_config["namespace"]

content {
name = each_item.value.name
value = each_item.value.value
labels = {
"app.kubernetes.io/managed-by" = "terraform-ssp-amazon-eks"
}
}
}
18 changes: 0 additions & 18 deletions modules/kubernetes-addons/spark-k8s-operator/outputs.tf
Original file line number Diff line number Diff line change
@@ -1,21 +1,3 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: MIT-0
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of this
* software and associated documentation files (the "Software"), to deal in the Software
* without restriction, including without limitation the rights to use, copy, modify,
* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

output "argocd_gitops_config" {
description = "Configuration used for managing the add-on with ArgoCD"
value = var.manage_via_gitops ? local.argocd_gitops_config : null
Expand Down
Loading

0 comments on commit ea0922f

Please sign in to comment.