diff --git a/charts/digitalhub/Chart.yaml b/charts/digitalhub/Chart.yaml index 56ed702d..596f4bcc 100644 --- a/charts/digitalhub/Chart.yaml +++ b/charts/digitalhub/Chart.yaml @@ -7,8 +7,8 @@ maintainers: url: https://github.com/ffais - name: calcagiara url: https://github.com/Calcagiara -version: "0.8.0" -appVersion: "0.8.0" +version: "0.9.0-beta1" +appVersion: "0.9.0" dependencies: - name: apigw-operator version: "0.1.8" @@ -19,7 +19,7 @@ dependencies: repository: https://helm.coder.com/v2 condition: coder.enabled - name: core - version: "0.2.17" + version: "0.2.18" repository: https://scc-digitalhub.github.io/digitalhub/ condition: core.enabled - name: kubernetes-resource-manager @@ -35,11 +35,11 @@ dependencies: repository: "https://helm.twun.io" condition: docker-registry.enabled - name: ext-postgres-operator - version: "1.2.4" + version: "1.2.5" repository: https://scc-digitalhub.github.io/digitalhub/ condition: ext-postgres-operator.enabled - name: kubeflow-pipelines - version: "0.1.0" + version: "0.1.1" repository: https://scc-digitalhub.github.io/digitalhub/ condition: kubeflow-pipelines.enabled - name: minio diff --git a/charts/digitalhub/confs/dashboard/env.js b/charts/digitalhub/confs/dashboard/env.js index c19b7455..35737d1e 100644 --- a/charts/digitalhub/confs/dashboard/env.js +++ b/charts/digitalhub/confs/dashboard/env.js @@ -1,5 +1,5 @@ {{ if .Values.dashboard.oidc.enabled -}} -window.env = {"VITE_OIDC_CONFIG":JSON.stringify({"accessTokenExpiringNotificationTime": "3570", "authority": "{{ .Values.dashboard.oidc.config.issuer }}", "clientId": "{{ .Values.dashboard.oidc.audience.clientId }}", "redirectUri": "http://{{ include "digitalhub.oidcDashboardEndpoint" . }}/oidc-callback", "responseType": "code", "scope": "openid profile email", "automaticSilentRenew": "false", "automaticSilentSignin": "false", "post_logout_redirect_uri": "http://{{ include "digitalhub.oidcDashboardEndpoint" . }}"}), "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.8"} +window.env = {"VITE_OIDC_CONFIG":JSON.stringify({"accessTokenExpiringNotificationTime": "3570", "authority": "{{ .Values.dashboard.oidc.config.issuer }}", "clientId": "{{ .Values.dashboard.oidc.audience.clientId }}", "redirectUri": "http://{{ include "digitalhub.oidcDashboardEndpoint" . }}/oidc-callback", "responseType": "code", "scope": "openid profile email", "automaticSilentRenew": "false", "automaticSilentSignin": "false", "post_logout_redirect_uri": "http://{{ include "digitalhub.oidcDashboardEndpoint" . }}"}), "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.9"} {{- else -}} -window.env = {"VITE_OIDC_CONFIG": null, "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.8"} +window.env = {"VITE_OIDC_CONFIG": null, "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.9"} {{- end }} diff --git a/charts/digitalhub/confs/tests/init-python-script.sh b/charts/digitalhub/confs/tests/init-python-script.sh new file mode 100644 index 00000000..df4fc368 --- /dev/null +++ b/charts/digitalhub/confs/tests/init-python-script.sh @@ -0,0 +1,5 @@ +#!/bin/sh +cp home/src/$TEST_SCRIPT home/git/digitalhub-tutorials/$TEST_FOLDER/$TEST_SCRIPT +cd home/git/digitalhub-tutorials/$TEST_FOLDER +pip install --cache-dir home/pipcache digitalhub[full] digitalhub-runtime-python digitalhub-runtime-container digitalhub-runtime-dbt digitalhub-runtime-kfp digitalhub-runtime-modelserve requests-oauthlib +python $TEST_SCRIPT diff --git a/charts/digitalhub/confs/tests/python-test/s1-etl.py b/charts/digitalhub/confs/tests/python-test/s1-etl.py new file mode 100644 index 00000000..080c9e4e --- /dev/null +++ b/charts/digitalhub/confs/tests/python-test/s1-etl.py @@ -0,0 +1,58 @@ +import sys +import digitalhub as dh +from time import time, sleep +import os +from oauthlib.oauth2 import BackendApplicationClient +from requests_oauthlib import OAuth2Session + + +def poller(run): + + start = time() + max_time = 15 * 60 # 15 minutes + + while True: + + if (time() - start) > max_time: + raise Exception(f"Timed out waiting: run status is {run.status.state}") + + run.refresh() + + if run.status.state == "ERROR": + raise Exception(f"Something got wrong with run: {run.status.state} - {run.status.message}") + + if run.status.state == "COMPLETED": + print("Run finished.") + sys.exit(0) + + if run.status.state == "STOPPED": + print("Run stopped.") + sys.exit(1) + + sleep(5) + + +def main(): + if "CORE_CLIENT_ID" in os.environ: + # Get Core Token + client_id = os.getenv("CORE_CLIENT_ID") + client_secret = os.getenv("CORE_CLIENT_SECRET") + scope = 'tenant1-core' + client = BackendApplicationClient(client_id=client_id) + oauth = OAuth2Session(client=client, scope=scope) + token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope) + os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"] + + # Load project + proj = dh.import_project("project-etl-ci.yml") + + URL = "https://opendata.comune.bologna.it/api/explore/v2.1/catalog/datasets/rilevazione-flusso-veicoli-tramite-spire-anno-2023/exports/csv?lang=it&timezone=Europe%2FRome&use_labels=true&delimiter=%3B" + di= proj.new_dataitem(name="url_data_item",kind="table",path=URL) + workflow_run = proj.run('pipeline', parameters={"url": di.key}) + + # Wait for run to finish + poller(workflow_run) + + +if __name__ == "__main__": + main() diff --git a/charts/digitalhub/confs/tests/python-test/s2-dbt.py b/charts/digitalhub/confs/tests/python-test/s2-dbt.py new file mode 100644 index 00000000..77f27b80 --- /dev/null +++ b/charts/digitalhub/confs/tests/python-test/s2-dbt.py @@ -0,0 +1,58 @@ +import sys +import digitalhub as dh +from time import time, sleep +import os +from oauthlib.oauth2 import BackendApplicationClient +from requests_oauthlib import OAuth2Session + + +def poller(run): + + start = time() + max_time = 15 * 60 # 15 minutes + + while True: + + if (time() - start) > max_time: + raise Exception(f"Timed out waiting: run status is {run.status.state}") + + run.refresh() + + if run.status.state == "ERROR": + raise Exception(f"Something got wrong with run: {run.status.state} - {run.status.message}") + + if run.status.state == "COMPLETED": + print("Run finished.") + sys.exit(0) + + if run.status.state == "STOPPED": + print("Run stopped.") + sys.exit(1) + + sleep(5) + + +def main(): + if "CORE_CLIENT_ID" in os.environ: + # Get Core Token + client_id = os.getenv("CORE_CLIENT_ID") + client_secret = os.getenv("CORE_CLIENT_SECRET") + scope = 'tenant1-core' + client = BackendApplicationClient(client_id=client_id) + oauth = OAuth2Session(client=client, scope=scope) + token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope) + os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"] + + # Load project + proj = dh.import_project("project-dbt-ci.yml") + + url = "https://gist.githubusercontent.com/kevin336/acbb2271e66c10a5b73aacf82ca82784/raw/e38afe62e088394d61ed30884dd50a6826eee0a8/employees.csv" + di_url = proj.new_dataitem(name="url_data_item",kind="table",path=url) + workflow_run = proj.run('pipeline_dbt', parameters={"url": di_url.key}) + + # Wait for run to finish + poller(workflow_run) + + +if __name__ == "__main__": + main() diff --git a/charts/digitalhub/confs/tests/python-test/s3-scikit-learn.py b/charts/digitalhub/confs/tests/python-test/s3-scikit-learn.py new file mode 100644 index 00000000..2448f0b5 --- /dev/null +++ b/charts/digitalhub/confs/tests/python-test/s3-scikit-learn.py @@ -0,0 +1,56 @@ +import sys +import digitalhub as dh +from time import time, sleep +import os +from oauthlib.oauth2 import BackendApplicationClient +from requests_oauthlib import OAuth2Session + + +def poller(run): + + start = time() + max_time = 15 * 60 # 15 minutes + + while True: + + if (time() - start) > max_time: + raise Exception(f"Timed out waiting: run status is {run.status.state}") + + run.refresh() + + if run.status.state == "ERROR": + raise Exception(f"Something got wrong with run: {run.status.state} - {run.status.message}") + + if run.status.state == "COMPLETED": + print("Run finished.") + sys.exit(0) + + if run.status.state == "STOPPED": + print("Run stopped.") + sys.exit(1) + + sleep(5) + + +def main(): + if "CORE_CLIENT_ID" in os.environ: + # Get Core Token + client_id = os.getenv("CORE_CLIENT_ID") + client_secret = os.getenv("CORE_CLIENT_SECRET") + scope = 'tenant1-core' + client = BackendApplicationClient(client_id=client_id) + oauth = OAuth2Session(client=client, scope=scope) + token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope) + os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"] + + # Load project + proj = dh.import_project("project-ml-ci.yml") + + workflow_run = proj.run('pipeline_ml') + + # Wait for run to finish + poller(workflow_run) + + +if __name__ == "__main__": + main() diff --git a/charts/digitalhub/templates/dashboard/deployment.yaml b/charts/digitalhub/templates/dashboard/deployment.yaml index 1175357d..a3fe603d 100644 --- a/charts/digitalhub/templates/dashboard/deployment.yaml +++ b/charts/digitalhub/templates/dashboard/deployment.yaml @@ -15,6 +15,7 @@ spec: metadata: annotations: checksum/config: {{ include (print $.Template.BasePath "/dashboard/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/dashboard/oidcconfig.yaml") . | sha256sum }} {{- with .Values.dashboard.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/digitalhub/templates/dashboard/oidcconfig.yaml b/charts/digitalhub/templates/dashboard/oidcconfig.yaml index a1cccb24..8766dd4e 100644 --- a/charts/digitalhub/templates/dashboard/oidcconfig.yaml +++ b/charts/digitalhub/templates/dashboard/oidcconfig.yaml @@ -21,7 +21,7 @@ spec: name: dashboard-oidc-secret stringData: env.js: |- - window.env = {"VITE_OIDC_CONFIG":JSON.stringify({"accessTokenExpiringNotificationTime": "3570", "authority": "{{ .Values.dashboard.oidc.config.issuer }}", "clientId": "{{ "{{" }} matrix.aacdashboardsecret.data.clientid | b64decode {{ "}}" }}", "redirectUri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}/oidc-callback", "responseType": "code", "scope": "openid profile email", "automaticSilentRenew": "false", "automaticSilentSignin": "false", "post_logout_redirect_uri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}"}), "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.8"} + window.env = {"VITE_OIDC_CONFIG":JSON.stringify({"accessTokenExpiringNotificationTime": "3570", "authority": "{{ .Values.dashboard.oidc.config.issuer }}", "clientId": "{{ "{{" }} matrix.aacdashboardsecret.data.clientid | b64decode {{ "}}" }}", "redirectUri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}/oidc-callback", "responseType": "code", "scope": "openid profile email", "automaticSilentRenew": "false", "automaticSilentSignin": "false", "post_logout_redirect_uri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}"}), "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.9"} {{- else -}} {{- $v := tpl (.Files.Get "confs/dashboard/env.js") . }} apiVersion: v1 diff --git a/charts/digitalhub/templates/oauth2-proxy/deployment.yaml b/charts/digitalhub/templates/oauth2-proxy/deployment.yaml index 411c7706..f423156f 100644 --- a/charts/digitalhub/templates/oauth2-proxy/deployment.yaml +++ b/charts/digitalhub/templates/oauth2-proxy/deployment.yaml @@ -13,6 +13,8 @@ spec: app: oauth-proxy-{{ $value.name }} template: metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/oauth2-proxy/configmap.yaml") . | sha256sum }} labels: app: oauth-proxy-{{ $value.name }} spec: diff --git a/charts/digitalhub/templates/tests/dhub-test/test-configmap.yaml b/charts/digitalhub/templates/tests/dhub-test/test-configmap.yaml index b2bc4c9e..d0935e46 100644 --- a/charts/digitalhub/templates/tests/dhub-test/test-configmap.yaml +++ b/charts/digitalhub/templates/tests/dhub-test/test-configmap.yaml @@ -5,7 +5,14 @@ metadata: data: {{- $files := .Files }} {{- range $key, $value := .Files }} -{{- if hasPrefix "confs/tests/python-test/python-test" $key }} +{{- if hasPrefix "confs/tests/python-test/" $key }} {{ $key | trimPrefix "confs/tests/python-test/" }}: {{ $files.Get $key | quote }} {{- end }} {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "digitalhub.fullname" . }}-python-script-test +data: +{{ (.Files.Glob "confs/tests/init-python-script.sh").AsConfig | indent 2 }} diff --git a/charts/digitalhub/templates/tests/python-test.yaml b/charts/digitalhub/templates/tests/python-test.yaml index c83163ad..0a75fe39 100644 --- a/charts/digitalhub/templates/tests/python-test.yaml +++ b/charts/digitalhub/templates/tests/python-test.yaml @@ -1,7 +1,7 @@ {{- $files := .Files }} {{- range $key, $value := .Files }} -{{- if hasPrefix "confs/tests/python-test/python-test" $key }} -{{- $testName := ($key | trimPrefix "confs/tests/python-test/python-test-" | trimSuffix ".py") }} +{{- if hasPrefix "confs/tests/python-test/" $key }} +{{- $testName := ($key | trimPrefix "confs/tests/python-test/" | trimSuffix ".py") }} apiVersion: batch/v1 kind: Job metadata: @@ -13,13 +13,31 @@ metadata: spec: template: spec: + initContainers: + - name: clone-repo + image: bitnami/git:latest + command: ["/bin/sh", "-c", "cd home/git/ && git clone https://github.com/scc-digitalhub/digitalhub-tutorials.git"] + volumeMounts: + - name: git-volume + mountPath: /home/git + readOnly: false + securityContext: + readOnlyRootFilesystem: false + runAsUser: 0 containers: - name: python - image: python:3.9-slim - command: ['sh', '-c', 'pip install "digitalhub_runtime_{{$testName}}[local]{{ $.Values.core.testSuite }}" requests-oauthlib && python home/{{ $key | trimPrefix "confs/tests/python-test/" }}'] + image: python:3.10-slim + command: ['sh', './home/scripts/init-python-script.sh'] volumeMounts: - name: test-volume - mountPath: /home + mountPath: /home/src + readOnly: false + - name: test-python-script-volume + mountPath: /home/scripts/init-python-script.sh + readOnly: false + subPath: "init-python-script.sh" + - name: git-volume + mountPath: /home/git readOnly: false env: - name: "POSTGRES_USER" @@ -32,6 +50,10 @@ spec: secretKeyRef: name: "digitalhub-owner-user.database-postgres-cluster.credentials.postgresql.acid.zalan.do" key: "password" + - name: TEST_SCRIPT + value: {{ $key | trimPrefix "confs/tests/python-test/" }} + - name: TEST_FOLDER + value: {{ $key | trimPrefix "confs/tests/python-test/" | trimSuffix ".py" }} {{- if $.Values.core.authentication.openId.enabled }} - name: "CORE_CLIENT_ID" valueFrom: @@ -62,10 +84,15 @@ spec: - name: test-volume configMap: name: {{ include "digitalhub.fullname" $ }}-python-test + - name: test-python-script-volume + configMap: + name: {{ include "digitalhub.fullname" $ }}-python-script-test + - name: git-volume + emptyDir: {} securityContext: - runAsUser: 1000 - fsGroup: 100 - runAsGroup: 100 + runAsUser: 0 + fsGroup: 0 + runAsGroup: 0 restartPolicy: Never backoffLimit: 0 ---