feat(dhub): added more accurate tests for the platform, fixed dashboa…

…rd pod restart

charts/digitalhub/Chart.yaml

@@ -7,8 +7,8 @@ maintainers:
     url: https://github.com/ffais
   - name: calcagiara
     url: https://github.com/Calcagiara
-version: "0.8.0"
-appVersion: "0.8.0"
+version: "0.9.0-beta1"
+appVersion: "0.9.0"
 dependencies:
 - name: apigw-operator
   version: "0.1.8"
@@ -19,7 +19,7 @@ dependencies:
   repository: https://helm.coder.com/v2
   condition: coder.enabled
 - name: core
-  version: "0.2.17"
+  version: "0.2.18"
   repository: https://scc-digitalhub.github.io/digitalhub/
   condition: core.enabled
 - name: kubernetes-resource-manager
@@ -35,11 +35,11 @@ dependencies:
   repository: "https://helm.twun.io"
   condition: docker-registry.enabled
 - name: ext-postgres-operator
-  version: "1.2.4"
+  version: "1.2.5"
   repository: https://scc-digitalhub.github.io/digitalhub/
   condition: ext-postgres-operator.enabled
 - name: kubeflow-pipelines
-  version: "0.1.0"
+  version: "0.1.1"
   repository: https://scc-digitalhub.github.io/digitalhub/
   condition: kubeflow-pipelines.enabled
 - name: minio

charts/digitalhub/confs/tests/init-python-script.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+cp home/src/$TEST_SCRIPT home/git/digitalhub-tutorials/$TEST_FOLDER/$TEST_SCRIPT
+cd home/git/digitalhub-tutorials/$TEST_FOLDER
+pip install --cache-dir home/pipcache digitalhub[full] digitalhub-runtime-python digitalhub-runtime-container digitalhub-runtime-dbt digitalhub-runtime-kfp digitalhub-runtime-modelserve requests-oauthlib
+python $TEST_SCRIPT

charts/digitalhub/confs/tests/python-test/s1-etl.py

@@ -0,0 +1,58 @@
+import sys
+import digitalhub as dh
+from time import time, sleep
+import os
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2Session
+
+
+def poller(run):
+
+    start = time()
+    max_time = 15 * 60 # 15 minutes
+
+    while True:
+
+        if (time() - start) > max_time:
+            raise Exception(f"Timed out waiting: run status is {run.status.state}")
+
+        run.refresh()
+
+        if run.status.state == "ERROR":
+            raise Exception(f"Something got wrong with run: {run.status.state} - {run.status.message}")
+
+        if run.status.state == "COMPLETED":
+            print("Run finished.")
+            sys.exit(0)
+
+        if run.status.state == "STOPPED":
+            print("Run stopped.")
+            sys.exit(1)
+
+        sleep(5)
+
+
+def main():
+    if "CORE_CLIENT_ID" in os.environ:
+        # Get Core Token
+        client_id = os.getenv("CORE_CLIENT_ID")
+        client_secret = os.getenv("CORE_CLIENT_SECRET")
+        scope = 'tenant1-core'
+        client = BackendApplicationClient(client_id=client_id)
+        oauth = OAuth2Session(client=client, scope=scope)
+        token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope)
+        os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"]
+
+    # Load project
+    proj = dh.import_project("project-etl-ci.yml")
+
+    URL = "https://opendata.comune.bologna.it/api/explore/v2.1/catalog/datasets/rilevazione-flusso-veicoli-tramite-spire-anno-2023/exports/csv?lang=it&timezone=Europe%2FRome&use_labels=true&delimiter=%3B"
+    di= proj.new_dataitem(name="url_data_item",kind="table",path=URL)
+    workflow_run = proj.run('pipeline', parameters={"url": di.key})
+
+    # Wait for run to finish
+    poller(workflow_run)
+
+
+if __name__ == "__main__":
+    main()

charts/digitalhub/confs/tests/python-test/s2-dbt.py

@@ -0,0 +1,58 @@
+import sys
+import digitalhub as dh
+from time import time, sleep
+import os
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2Session
+
+
+def poller(run):
+
+    start = time()
+    max_time = 15 * 60 # 15 minutes
+
+    while True:
+
+        if (time() - start) > max_time:
+            raise Exception(f"Timed out waiting: run status is {run.status.state}")
+
+        run.refresh()
+
+        if run.status.state == "ERROR":
+            raise Exception(f"Something got wrong with run: {run.status.state} - {run.status.message}")
+
+        if run.status.state == "COMPLETED":
+            print("Run finished.")
+            sys.exit(0)
+
+        if run.status.state == "STOPPED":
+            print("Run stopped.")
+            sys.exit(1)
+
+        sleep(5)
+
+
+def main():
+    if "CORE_CLIENT_ID" in os.environ:
+        # Get Core Token
+        client_id = os.getenv("CORE_CLIENT_ID")
+        client_secret = os.getenv("CORE_CLIENT_SECRET")
+        scope = 'tenant1-core'
+        client = BackendApplicationClient(client_id=client_id)
+        oauth = OAuth2Session(client=client, scope=scope)
+        token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope)
+        os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"]
+
+    # Load project
+    proj = dh.import_project("project-dbt-ci.yml")
+
+    url = "https://gist.githubusercontent.com/kevin336/acbb2271e66c10a5b73aacf82ca82784/raw/e38afe62e088394d61ed30884dd50a6826eee0a8/employees.csv"
+    di_url = proj.new_dataitem(name="url_data_item",kind="table",path=url)
+    workflow_run = proj.run('pipeline_dbt', parameters={"url": di_url.key})
+
+    # Wait for run to finish
+    poller(workflow_run)
+
+
+if __name__ == "__main__":
+    main()

charts/digitalhub/confs/tests/python-test/s3-scikit-learn.py

@@ -0,0 +1,56 @@
+import sys
+import digitalhub as dh
+from time import time, sleep
+import os
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2Session
+
+
+def poller(run):
+
+    start = time()
+    max_time = 15 * 60 # 15 minutes
+
+    while True:
+
+        if (time() - start) > max_time:
+            raise Exception(f"Timed out waiting: run status is {run.status.state}")
+
+        run.refresh()
+
+        if run.status.state == "ERROR":
+            raise Exception(f"Something got wrong with run: {run.status.state} - {run.status.message}")
+
+        if run.status.state == "COMPLETED":
+            print("Run finished.")
+            sys.exit(0)
+
+        if run.status.state == "STOPPED":
+            print("Run stopped.")
+            sys.exit(1)
+
+        sleep(5)
+
+
+def main():
+    if "CORE_CLIENT_ID" in os.environ:
+        # Get Core Token
+        client_id = os.getenv("CORE_CLIENT_ID")
+        client_secret = os.getenv("CORE_CLIENT_SECRET")
+        scope = 'tenant1-core'
+        client = BackendApplicationClient(client_id=client_id)
+        oauth = OAuth2Session(client=client, scope=scope)
+        token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope)
+        os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"]
+
+    # Load project
+    proj = dh.import_project("project-ml-ci.yml")
+
+    workflow_run = proj.run('pipeline_ml')
+
+    # Wait for run to finish
+    poller(workflow_run)
+
+
+if __name__ == "__main__":
+    main()

charts/digitalhub/templates/dashboard/deployment.yaml

@@ -15,6 +15,7 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/dashboard/configmap.yaml") . | sha256sum }}
+        checksum/secret: {{ include (print $.Template.BasePath "/dashboard/oidcconfig.yaml") . | sha256sum }}
       {{- with .Values.dashboard.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

charts/digitalhub/templates/dashboard/oidcconfig.yaml

@@ -21,7 +21,7 @@ spec:
         name: dashboard-oidc-secret
       stringData:
         env.js: |-
-          window.env = {"VITE_OIDC_CONFIG":JSON.stringify({"accessTokenExpiringNotificationTime": "3570", "authority": "{{ .Values.dashboard.oidc.config.issuer }}", "clientId": "{{ "{{" }} matrix.aacdashboardsecret.data.clientid | b64decode {{ "}}" }}", "redirectUri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}/oidc-callback", "responseType": "code", "scope": "openid profile email", "automaticSilentRenew": "false", "automaticSilentSignin": "false", "post_logout_redirect_uri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}"}), "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.8"}
+          window.env = {"VITE_OIDC_CONFIG":JSON.stringify({"accessTokenExpiringNotificationTime": "3570", "authority": "{{ .Values.dashboard.oidc.config.issuer }}", "clientId": "{{ "{{" }} matrix.aacdashboardsecret.data.clientid | b64decode {{ "}}" }}", "redirectUri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}/oidc-callback", "responseType": "code", "scope": "openid profile email", "automaticSilentRenew": "false", "automaticSilentSignin": "false", "post_logout_redirect_uri": "https://{{ include "digitalhub.oidcDashboardEndpoint" . }}"}), "VITE_PLATFORM_TITLE": "OltreAI", "VITE_PLATFORM_VERSION": "0.9"}
 {{- else -}}
 {{- $v := tpl (.Files.Get "confs/dashboard/env.js") . }}
 apiVersion: v1

charts/digitalhub/templates/oauth2-proxy/deployment.yaml

@@ -13,6 +13,8 @@ spec:
       app: oauth-proxy-{{ $value.name }}
   template:
     metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/oauth2-proxy/configmap.yaml") . | sha256sum }}
       labels:
         app: oauth-proxy-{{ $value.name }}
     spec:

charts/digitalhub/templates/tests/dhub-test/test-configmap.yaml

@@ -5,7 +5,14 @@ metadata:
 data:
 {{- $files := .Files }}
 {{- range $key, $value := .Files }}
-{{- if hasPrefix "confs/tests/python-test/python-test" $key }}
+{{- if hasPrefix "confs/tests/python-test/" $key }}
   {{ $key | trimPrefix "confs/tests/python-test/" }}: {{ $files.Get $key | quote }}
 {{- end }}
 {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "digitalhub.fullname" . }}-python-script-test
+data:
+{{ (.Files.Glob "confs/tests/init-python-script.sh").AsConfig | indent 2 }}

charts/digitalhub/templates/tests/python-test.yaml

@@ -1,7 +1,7 @@
 {{- $files := .Files }}
 {{- range $key, $value := .Files }}
-{{- if hasPrefix "confs/tests/python-test/python-test" $key }}
-{{- $testName := ($key | trimPrefix "confs/tests/python-test/python-test-" | trimSuffix ".py") }}
+{{- if hasPrefix "confs/tests/python-test/" $key }}
+{{- $testName := ($key | trimPrefix "confs/tests/python-test/" | trimSuffix ".py") }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -13,13 +13,31 @@ metadata:
 spec:
   template:
     spec:
+      initContainers:
+        - name: clone-repo
+          image: bitnami/git:latest
+          command: ["/bin/sh", "-c", "cd home/git/ && git clone https://github.com/scc-digitalhub/digitalhub-tutorials.git"]
+          volumeMounts:
+            - name: git-volume
+              mountPath: /home/git
+              readOnly: false
+          securityContext:
+            readOnlyRootFilesystem: false
+            runAsUser: 0
       containers:
         - name: python
-          image: python:3.9-slim
-          command: ['sh', '-c', 'pip install "digitalhub_runtime_{{$testName}}[local]{{ $.Values.core.testSuite }}" requests-oauthlib && python home/{{ $key | trimPrefix "confs/tests/python-test/" }}']
+          image: python:3.10-slim
+          command: ['sh', './home/scripts/init-python-script.sh']
           volumeMounts:
             - name: test-volume
-              mountPath: /home
+              mountPath: /home/src
+              readOnly: false
+            - name: test-python-script-volume
+              mountPath: /home/scripts/init-python-script.sh
+              readOnly: false
+              subPath: "init-python-script.sh"
+            - name: git-volume
+              mountPath: /home/git
               readOnly: false
           env:
             - name: "POSTGRES_USER"
@@ -32,6 +50,10 @@ spec:
                 secretKeyRef:
                   name: "digitalhub-owner-user.database-postgres-cluster.credentials.postgresql.acid.zalan.do"
                   key: "password"
+            - name: TEST_SCRIPT
+              value: {{ $key | trimPrefix "confs/tests/python-test/" }}
+            - name: TEST_FOLDER
+              value: {{ $key | trimPrefix "confs/tests/python-test/" | trimSuffix ".py" }}
             {{- if $.Values.core.authentication.openId.enabled }}
             - name: "CORE_CLIENT_ID"
               valueFrom:
@@ -62,10 +84,15 @@ spec:
         - name: test-volume
           configMap:
             name: {{ include "digitalhub.fullname" $ }}-python-test
+        - name: test-python-script-volume
+          configMap:
+            name: {{ include "digitalhub.fullname" $ }}-python-script-test
+        - name: git-volume
+          emptyDir: {}      
       securityContext:
-        runAsUser: 1000
-        fsGroup: 100
-        runAsGroup: 100
+        runAsUser: 0
+        fsGroup: 0
+        runAsGroup: 0
       restartPolicy: Never
   backoffLimit: 0
 ---