Merge branch 'master' into feat/non-eks-manifestservice

uc-cdis · Feb 24, 2025 · 2609752 · 2609752
2 parents 0862acb + 754bb7d
commit 2609752
Show file tree

Hide file tree

Showing 30 changed files with 202 additions and 118 deletions.
diff --git a/NOTICE b/NOTICE
@@ -0,0 +1,7 @@
+Copyright 2022-2025 University of Chicago
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
diff --git a/README.md b/README.md
@@ -7,28 +7,28 @@ Helm charts for deploying [Gen3](https://gen3.org) on any kubernetes cluster.
 
 # Deploying gen3 with helm
 
-## TL;DR 
+## TL;DR
 ```
 helm repo add gen3 https://helm.gen3.org
 helm repo update
-helm upgrade --install gen3 gen3/gen3 -f ./values.yaml 
+helm upgrade --install gen3 gen3/gen3 -f ./values.yaml
 ```
 
-For more information on how to deploy Gen3 with helm, please see the [Gen3 Example Deployment Guide](https://docs.gen3.org/docs/Deployment/Example%20Deployment)
+For more information on how to deploy Gen3 with helm, please see the [Gen3 Example Deployment Guide](https://docs.gen3.org/gen3-resources/operator-guide/helm/)
 
 https://docs.gen3.org
 
 
 ## Configuration
 
-For a full set of configuration options see the [CONFIGURATION.md](./docs/CONFIGURATION.md) for a more in depth instructions on how to configure each service. 
+For a full set of configuration options see the [CONFIGURATION.md](./docs/CONFIGURATION.md) for a more in depth instructions on how to configure each service.
 
-There's also an auto-generated table of basic configuration options here: 
+There's also an auto-generated table of basic configuration options here:
 
-[README.md for gen3 chart](./helm/gen3/README.md) (auto-generated documentation) or 
+[README.md for gen3 chart](./helm/gen3/README.md) (auto-generated documentation) or
 
 
-To see documentation around setting up gen3 developer environments see [our Example Deployment](https://docs.gen3.org/docs/Deployment/Example%20Deployment/).
+To see documentation around setting up gen3 developer environments see [our Example Deployment](https://docs.gen3.org/gen3-resources/operator-guide/helm/helm-deploy-example/).
 
 
 Use the following as a template for your `values.yaml` file for a minimum deployment of gen3 using these helm charts.
@@ -39,19 +39,19 @@ Use the following as a template for your `values.yaml` file for a minimum deploy
 global:
   hostname: example-commons.com
 
-fence: 
+fence:
   FENCE_CONFIG:
-    # Any fence-config overrides here. 
+    # Any fence-config overrides here.
 ```
 
 
 ## Gen3 Login Options
-Gen3 does not have any IDP, but can integrate with many. We will cover Google login here, but refer to the fence documentation for additional options. 
+Gen3 does not have any IDP, but can integrate with many. We will cover Google login here, but refer to the fence documentation for additional options.
 
 TL/DR: At minimum to have google logins working you need to set these settings in your `values.yaml` file
 
 ```
-fence: 
+fence:
   FENCE_CONFIG:
     OPENID_CONNECT:
       google:
@@ -62,7 +62,7 @@ fence:
 
 #### Google login generation
 
-You need to set up a google credential for google login as that's the default enabled option in fence. 
+You need to set up a google credential for google login as that's the default enabled option in fence.
 
 
 The following steps explain how to create credentials for your gen3
@@ -76,9 +76,9 @@ Name your OAuth 2.0 client and click Create.
 
 For `Authorized Javascript Origins` add `https://<hostname>`
 
-For `"Authorized redirect URIs"` add  `https://<hostname>/user/login/google/login/` 
+For `"Authorized redirect URIs"` add  `https://<hostname>/user/login/google/login/`
 
-After configuration is complete, take note of the client ID that was created. You will need the client ID and client secret to complete the next steps. 
+After configuration is complete, take note of the client ID that was created. You will need the client ID and client secret to complete the next steps.
 
 
 # Troubleshooting

diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.29
+version: 0.1.30
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/fence/README.md b/helm/fence/README.md
@@ -1,6 +1,6 @@
 # fence
 
-![Version: 0.1.29](https://img.shields.io/badge/Version-0.1.29-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.30](https://img.shields.io/badge/Version-0.1.30-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Fence
 
@@ -184,9 +184,10 @@ A Helm chart for gen3 Fence
 | serviceAccount.create | bool | `true` | Specifies whether a service account should be created. |
 | serviceAccount.name | string | `"fence-sa"` | The name of the service account |
 | tolerations | list | `[]` | Tolerations for the pods |
-| usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"schedule":"*/30 * * * *","slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. |
+| usersync | map | `{"addDbgap":false,"custom_image":null,"env":null,"onlyDbgap":false,"schedule":"*/30 * * * *","slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. |
 | usersync.addDbgap | bool | `false` | Force attempting a dbgap sync if "true", falls back on user.yaml |
 | usersync.custom_image | string | `nil` | To set a custom image for pulling the user.yaml file from S3. Default is the Gen3 Awshelper image. |
+| usersync.env | list | `nil` | Environment variables to pass to the job. |
 | usersync.onlyDbgap | bool | `false` | Forces ONLY a dbgap sync if "true", IGNORING user.yaml |
 | usersync.schedule | string | `"*/30 * * * *"` | The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. |
 | usersync.slack_send_dbgap | bool | `false` | Will echo what files we are seeing on dbgap ftp to Slack. |

diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml
@@ -71,15 +71,18 @@ spec:
               image: {{ .Values.usersync.custom_image | default "quay.io/cdis/awshelper:master" }}
               imagePullPolicy: Always
               env:
-              - name: gen3Env
-                valueFrom:
+                - name: gen3Env
+                  valueFrom:
                     configMapKeyRef:
                       name: manifest-global
                       key: hostname
-              - name: userYamlS3Path
-                value: {{ .Values.usersync.userYamlS3Path | quote }}
-              - name: slackWebHook
-                value: {{ .Values.usersync.slack_webhook | quote }}
+                - name: userYamlS3Path
+                  value: {{ .Values.usersync.userYamlS3Path | quote }}
+                - name: slackWebHook
+                  value: {{ .Values.usersync.slack_webhook | quote }}
+                {{- with .Values.usersync.env }}
+                {{- toYaml . | nindent 16 }}
+                {{- end }}
               volumeMounts:
                 - name: user-yaml
                   mountPath: /var/www/fence
@@ -131,17 +134,22 @@ spec:
             image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
             imagePullPolicy: Always
             env:
-            - name: SYNC_FROM_DBGAP
-              value: {{ .Values.usersync.syncFromDbgap | quote }}
-            - name: ADD_DBGAP
-              value: {{ .Values.usersync.addDbgap | quote }}
-            - name: ONLY_DBGAP
-              value: {{ .Values.usersync.onlyDbgap | quote }}
-            - name: SLACK_SEND_DBGAP
-              value: {{ .Values.usersync.slack_send_dbgap | quote }}
-            - name: slackWebHook
-              value: {{ .Values.usersync.slack_webhook | quote }}
-            {{- toYaml .Values.env | nindent 12 }}
+              - name: SYNC_FROM_DBGAP
+                value: {{ .Values.usersync.syncFromDbgap | quote }}
+              - name: ADD_DBGAP
+                value: {{ .Values.usersync.addDbgap | quote }}
+              - name: ONLY_DBGAP
+                value: {{ .Values.usersync.onlyDbgap | quote }}
+              - name: SLACK_SEND_DBGAP
+                value: {{ .Values.usersync.slack_send_dbgap | quote }}
+              - name: slackWebHook
+                value: {{ .Values.usersync.slack_webhook | default "" | quote }}
+              {{- with .Values.env }}
+              {{- toYaml . | nindent 14 }}
+              {{- end }}
+              {{- with .Values.usersync.env }}
+              {{- toYaml . | nindent 14 }}
+              {{- end }}
             volumeMounts:
               - name: shared-data
                 mountPath: /mnt/shared

diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml
@@ -118,6 +118,8 @@ usersync:
   slack_webhook: None
   # -- (bool) Will echo what files we are seeing on dbgap ftp to Slack.
   slack_send_dbgap: false
+  # -- (list) Environment variables to pass to the job.
+  env:
 
 # -- (map) Secret information for Usersync and External Secrets.
 secrets:

diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml
@@ -36,7 +36,7 @@ dependencies:
     repository: "file://../frontend-framework"
     condition: frontend-framework.enabled
   - name: fence
-    version: 0.1.29
+    version: 0.1.30
     repository: "file://../fence"
     condition: fence.enabled
   - name: guppy
@@ -60,7 +60,7 @@ dependencies:
     repository: "file://../metadata"
     condition: metadata.enabled
   - name: peregrine
-    version: 0.1.17
+    version: 0.1.18
     repository: "file://../peregrine"
     condition: peregrine.enabled
   - name: portal
@@ -72,19 +72,19 @@ dependencies:
     repository: "file://../requestor"
     condition: requestor.enabled
   - name: revproxy
-    version: 0.1.19
+    version: 0.1.22
     repository: "file://../revproxy"
     condition: revproxy.enabled
   - name: sheepdog
     version: 0.1.20
     repository: "file://../sheepdog"
     condition: sheepdog.enabled
   - name: ssjdispatcher
-    version: 0.1.16
+    version: 0.1.18
     repository: "file://../ssjdispatcher"
     condition: ssjdispatcher.enabled
   - name: sower
-    version: 0.1.16
+    version: 0.1.17
     condition: sower.enabled
     repository: "file://../sower"
   - name: wts
@@ -128,7 +128,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.58
+version: 0.1.61
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/gen3/README.md b/helm/gen3/README.md
@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.58](https://img.shields.io/badge/Version-0.1.58-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.61](https://img.shields.io/badge/Version-0.1.61-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -25,7 +25,7 @@ Helm chart to deploy Gen3 Data Commons
 | file://../aws-es-proxy | aws-es-proxy | 0.1.13 |
 | file://../common | common | 0.1.16 |
 | file://../etl | etl | 0.1.10 |
-| file://../fence | fence | 0.1.29 |
+| file://../fence | fence | 0.1.30 |
 | file://../frontend-framework | frontend-framework | 0.1.6 |
 | file://../gen3-network-policies | gen3-network-policies | 0.1.2 |
 | file://../guppy | guppy | 0.1.17 |
@@ -34,13 +34,13 @@ Helm chart to deploy Gen3 Data Commons
 | file://../manifestservice | manifestservice | 0.1.18 |
 | file://../metadata | metadata | 0.1.17 |
 | file://../neuvector | neuvector | 0.1.2 |
-| file://../peregrine | peregrine | 0.1.17 |
+| file://../peregrine | peregrine | 0.1.18 |
 | file://../portal | portal | 0.1.26 |
 | file://../requestor | requestor | 0.1.16 |
-| file://../revproxy | revproxy | 0.1.19 |
+| file://../revproxy | revproxy | 0.1.22 |
 | file://../sheepdog | sheepdog | 0.1.20 |
-| file://../sower | sower | 0.1.16 |
-| file://../ssjdispatcher | ssjdispatcher | 0.1.16 |
+| file://../sower | sower | 0.1.17 |
+| file://../ssjdispatcher | ssjdispatcher | 0.1.18 |
 | file://../wts | wts | 0.1.18 |
 | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 |
 | https://helm.elastic.co | elasticsearch | 7.10.2 |

diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.17
+version: 0.1.18
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/peregrine/README.md b/helm/peregrine/README.md
@@ -1,6 +1,6 @@
 # peregrine
 
-![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.18](https://img.shields.io/badge/Version-0.1.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Peregrine service
 

diff --git a/helm/peregrine/templates/deployment.yaml b/helm/peregrine/templates/deployment.yaml
@@ -44,6 +44,9 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
+          {{- with .Values.env }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
             - name: FENCE_DB_USER
               valueFrom:
                 secretKeyRef:
@@ -149,7 +152,7 @@ spec:
             - name: GEN3_SIDECAR
               value: "False"
             - name: CONF_HOSTNAME
-              value: {{ .Values.global.hostname }}
+              value: {{ .Values.global.hostname | quote }}
           {{- with .Values.volumeMounts }}
           volumeMounts:
             {{- toYaml . | nindent 10 }}

diff --git a/helm/revproxy/Chart.yaml b/helm/revproxy/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.19
+version: 0.1.22
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md
@@ -1,6 +1,6 @@
 # revproxy
 
-![Version: 0.1.19](https://img.shields.io/badge/Version-0.1.19-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.22](https://img.shields.io/badge/Version-0.1.22-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 revproxy
 
@@ -23,10 +23,11 @@ A Helm chart for gen3 revproxy
 | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl |
 | criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". |
 | fullnameOverride | string | `""` | Override the full name of the deployment. |
-| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"wafv2":{"enabled":false,"wafAclArn":null}}` | AWS configuration |
+| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"scheme":"internet-facing","wafv2":{"enabled":false,"wafAclArn":null}}` | AWS configuration |
 | global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. |
 | global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. |
 | global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. |
+| global.aws.scheme | string | `"internet-facing"` | internal or internet-facing |
 | global.aws.wafv2 | map | `{"enabled":false,"wafAclArn":null}` | WAF configuration |
 | global.aws.wafv2.enabled | bool | `false` | Set to true if using AWS WAFv2 |
 | global.aws.wafv2.wafAclArn | string | `nil` | ARN for the WAFv2 ACL. |

diff --git a/helm/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf b/helm/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf
diff --git a/helm/revproxy/gen3.nginx.conf/argo-server.conf b/helm/revproxy/gen3.nginx.conf/argo-server.conf
@@ -0,0 +1,19 @@
+location /argo/ {
+    error_page 403 @errorworkspace;
+    set $authz_resource "/argo";
+    set $authz_method "access";
+    set $authz_service "argo";
+    # be careful - sub-request runs in same context as this request
+    auth_request /gen3-authz;
+
+    set $proxy_service  "argo";
+    set $upstream SERVICE_URL;
+
+    rewrite ^/argo/(.*) /$1 break;
+
+    proxy_set_header Connection '';
+    proxy_http_version 1.1;
+    chunked_transfer_encoding off;
+
+    proxy_pass $upstream;
+}
diff --git a/helm/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf b/helm/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf
@@ -0,0 +1,12 @@
+location /ai {
+    if ($csrf_check !~ ^ok-\S.+$) {
+    return 403 "failed csrf check";
+    }
+
+    set $proxy_service  "gen3-discovery-ai-service";
+    set $upstream http://gen3-discovery-ai-service$des_domain;
+    rewrite ^/ai/(.*) /$1 break;
+    proxy_pass $upstream;
+    proxy_redirect http://$host/ https://$host/ai/;
+    client_max_body_size 0;
+}