Merge pull request #526 from boozallen/502-add-irsa-support-for-spark…

…-infrastructure

#502 add irsa (IAM roles service account) support for spark-infrastructure

DRAFT_RELEASE_NOTES.md

@@ -1,5 +1,8 @@
 # Major Additions
 
+## Service account support for spark-infrastructure
+To have more flexible and secure way to authenticate AWS service, we add service account support for spark-infrastructure helm chart to enable the AWS IRSA (IAM Roles Service Account) authentication. See _**How to Upgrade**_ for more information.
+
 ## Path to Production Alignment
 To better align development processes with processes in CI/CD and higher environments, we no longer recommend using Tilt live-reloading.  As such, upgrading projects should consider narrowing the scope of their Tiltfile. See _**How to Upgrade**_ for more information.
 
@@ -94,6 +97,32 @@ To avoid duplicate docker builds, remove all the related `docker_build()` and `l
 
 ## Conditional Steps
 
+## AWS IRSA (IAM Roles Service Account) Authentication
+This is not a required step but a recommended way to authenticate AWS service
+1. [Create an IAM OIDC provider for your cluster](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html)
+2. Follow the [Assign IAM roles to Kubernetes service accounts](https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html) document but **skip** the step that creates the service account
+3. In the spark-infrastructure chart template, add the service account create configuration as below:
+
+aissemble-spark-history-chart
+```yaml
+aissemble-spark-history-chart:
+  serviceAccount:
+    name: service-account-name
+    enabled: true
+    metadata:
+      annotations:
+        # Ref: IAM roles arn from step 2
+        eks.amazonaws.com/role-arn: arn:aws:iam::aws-id:role/iam-role-name 
+```
+
+aissemble-thrift-server-chart:
+```yaml
+aissemble-thrift-server-chart:
+  deployment:
+    # service account name must match the service account name specified in the IAM roles trust relationships
+    serviceAccountName: service-account-name
+```
+
 ## Final Steps - Required for All Projects
 ### Finalizing the Upgrade
 1. Run `./mvnw org.technologybrewery.baton:baton-maven-plugin:baton-migrate` to apply the automatic migrations

extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/deployment.yaml

@@ -35,6 +35,11 @@ spec:
         {{ toYaml .Values.deployment.labels }}
         {{- end }}
     spec:
+      {{- if .Values.deployment.serviceAccountName }}
+      serviceAccountName: {{ .Values.deployment.serviceAccountName }}
+      {{- else if .Values.serviceAccount.create }}
+      serviceAccountName: {{ .Values.serviceAccount.name | default .Chart.Name }}
+      {{- end }}
       {{- if .Values.deployment.affinity }}
       affinity:
       {{- toYaml .Values.deployment.affinity | nindent 8 }}

extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/serviceaccount.yaml

@@ -0,0 +1,14 @@
+{{ if and .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccount.name | default .Chart.Name }}
+  {{ $otherdata := omit .Values.serviceAccount.metadata "name" "annotations" }}
+  {{- range $key, $value := $otherdata }}
+  {{ $key }}: {{ $value }}
+  {{- end }}
+  {{- with .Values.serviceAccount.metadata.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{ end }}

extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/serviceaccount_test.yaml

@@ -0,0 +1,54 @@
+suite: Spark History Service Account Test
+templates:
+  - serviceaccount.yaml
+tests:
+  - it: ServiceAccount does not exist by default
+    asserts:
+      - hasDocuments:
+          count: 0
+  - it: ServiceAccount should include appropriate default values if created
+    set:
+      serviceAccount:
+        create: true
+    asserts:
+      - containsDocument:
+          kind: ServiceAccount
+          apiVersion: v1
+      - equal:
+          path: metadata.name
+          value: aissemble-spark-history-chart-sa
+      - notExists:
+          path: metadata.annotations
+
+  - it: Should set values appropriately for the service account
+    set:
+      serviceAccount:
+        create: true
+        name: test
+        metadata:
+          namespace: unit-test
+          annotations:
+            eks.amazonaws.com/role-arn: arn:aws:iam::111222333444:role/test-access-role
+
+    asserts:
+      - equal:
+          path: metadata.name
+          value: test
+      - equal:
+          path: metadata.namespace
+          value: unit-test
+      - equal:
+          path: metadata.annotations["eks.amazonaws.com/role-arn"]
+          value: arn:aws:iam::111222333444:role/test-access-role
+
+  - it: Service account name uses Chart name if not set
+    set:
+      serviceAccount:
+        create: true
+        name: ""
+
+    asserts:
+      - equal:
+          path: metadata.name
+          value: aissemble-spark-history-chart
+

extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/values.yaml

@@ -62,3 +62,9 @@ eventVolume:
   volumePathOnNode: /tmp
 
 sparkConf: |-
+
+serviceAccount:
+  create: false
+  name: "aissemble-spark-history-chart-sa"
+  metadata:
+    annotations: {}