authorize requests to ml-pipeline endpoint if contains any trusted pr…

…incipals (kubeflow#2753)

* authorize requests to ml-pipeline endpoint if contains any trusted principals

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* initial test .github/workflows/pipeline_run_from_notebook.yaml

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* add Install KF Multi Tenancy

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* deploy admission-webhook as well

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* wait for notebook crd to have 1 ready replicas

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* add timeout 300s

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* add tests/gh-actions/run_and_wait_kubeflow_pipeline.py

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* install kubeflow pipelines

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* debug by getting pods

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* cosmetical changes

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* add ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* don't wait for status.phase succeeded

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* print some debug if pipeline run finished with failure

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* Accept failure of kf pipeline run with a note

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

* Revert "Accept failure of kf pipeline run with a note"

This reverts commit 32323a4.

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

---------

Signed-off-by: Krzysztof Romanowski <krzysztof.romanowski94@gmail.com>

.github/workflows/pipeline_run_from_notebook.yaml

@@ -0,0 +1,81 @@
+name: Create Pipeline Run from Kubeflow Notebook
+on:
+  pull_request:
+    paths:
+      - .github/workflows/pipeline_run_from_notebook.yaml
+      - apps/jupyter/notebook-controller/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_kind.sh
+      - tests/gh-actions/install_kustomize.sh
+      - tests/gh-actions/install_istio.sh
+      - common/istio*/**
+      - common/oidc-client/**
+      - apps/jupyter/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install KinD
+      run: ./tests/gh-actions/install_kind.sh
+
+    - name: Create KinD Cluster
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
+
+    - name: Install kustomize
+      run: ./tests/gh-actions/install_kustomize.sh
+
+    - name: Install Istio with ext auth
+      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+
+    - name: Install cert-manager
+      run: ./tests/gh-actions/install_cert_manager.sh
+
+    - name: Create kubeflow namespace
+      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
+
+    - name: Install kubeflow-istio-resources
+      run: kustomize build common/istio-1-22/kubeflow-istio-resources/base | kubectl apply -f -
+
+    - name: Install KF Pipelines
+      run: ./tests/gh-actions/install_pipelines.sh
+
+    - name: Install KF Multi Tenancy
+      run: ./tests/gh-actions/install_multi_tenancy.sh
+
+    - name: Build & Apply manifests
+      run: |
+        kustomize build apps/jupyter/jupyter-web-app/upstream/overlays/istio/ | kubectl apply -f -
+        kustomize build apps/jupyter/notebook-controller/upstream/overlays/kubeflow/ | kubectl apply -f -
+        kustomize build apps/admission-webhook/upstream/overlays/cert-manager | kubectl apply -f -
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 300s \
+          --field-selector=status.phase!=Succeeded
+
+    - name: Create KF Profile
+      run: kustomize build common/user-namespace/base | kubectl apply -f -
+
+    - name: Apply PodDefaults to access ml-pipeline with projected token
+      run: kubectl apply -f tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml
+
+    - name: Create Kubeflow Notebook with PodDefaults
+      run: |
+        kubectl apply -f tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml
+        kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 \
+          -f tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml \
+          --timeout 300s
+
+    - name: Wait for the kubeflow-m2m-oidc-configurator Job
+      run: |
+        ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
+
+    - name: Copy and execute the pipeline run script in KF Notebook
+      run: |
+        kubectl -n kubeflow-user-example-com cp \
+          ./tests/gh-actions/run_and_wait_kubeflow_pipeline.py \
+          test-0:/home/jovyan/run_and_wait_kubeflow_pipeline.py
+
+        kubectl -n kubeflow-user-example-com exec -ti \
+          test-0 -- python /home/jovyan/run_and_wait_kubeflow_pipeline.py

apps/pipeline/upstream/base/installs/multi-user/istio-authorization-config.yaml

@@ -32,10 +32,9 @@ spec:
         - cluster.local/ns/kubeflow/sa/ml-pipeline-scheduledworkflow
         - cluster.local/ns/kubeflow/sa/ml-pipeline-viewer-crd-service-account
         - cluster.local/ns/kubeflow/sa/kubeflow-pipelines-cache
-  # For user workloads, which cannot user http headers for authentication
-  - when:
-    - key: request.headers[kubeflow-userid]
-      notValues: ['*']
+  - from:
+    - source:
+        requestPrincipals: ["*"]  # allow access by any trusted principal
 ---
 apiVersion: security.istio.io/v1beta1
 kind: AuthorizationPolicy

tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml

@@ -0,0 +1,27 @@
+apiVersion: kubeflow.org/v1
+kind: Notebook
+metadata:
+  annotations:
+    notebooks.kubeflow.org/creator: user@example.com
+    notebooks.kubeflow.org/server-type: jupyter
+  generation: 1
+  labels:
+    access-ml-pipeline: "true"
+    app: test
+  name: test
+  namespace: kubeflow-user-example-com
+spec:
+  template:
+    spec:
+      containers:
+      - name: test
+        image: kubeflownotebookswg/jupyter-scipy:v1.9.0-rc.0
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            cpu: "0.6"
+            memory: 1.2Gi
+          requests:
+            cpu: "0.5"
+            memory: 1Gi
+      serviceAccountName: default-editor

tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml

@@ -0,0 +1,25 @@
+apiVersion: kubeflow.org/v1alpha1
+kind: PodDefault
+metadata:
+  name: access-ml-pipeline
+  namespace: kubeflow-user-example-com
+spec:
+  desc: Allow access to Kubeflow Pipelines
+  selector:
+    matchLabels:
+      access-ml-pipeline: "true"
+  env:
+    - name: KF_PIPELINES_SA_TOKEN_PATH
+      value: /var/run/secrets/kubeflow/pipelines/token
+  volumes:
+    - name: volume-kf-pipeline-token
+      projected:
+        sources:
+          - serviceAccountToken:
+              path: token
+              expirationSeconds: 7200
+              audience: pipelines.kubeflow.org
+  volumeMounts:
+    - mountPath: /var/run/secrets/kubeflow/pipelines
+      name: volume-kf-pipeline-token
+      readOnly: true

tests/gh-actions/run_and_wait_kubeflow_pipeline.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+from kfp import dsl
+import kfp
+from time import sleep
+import subprocess
+import logging
+import sys
+from datetime import datetime, timezone
+
+logger = logging.getLogger("run_and_wait_for_pipeline")
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.DEBUG,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+
+
+client = kfp.Client()
+experiment_name = "my-experiment"
+experiment_namespace = "kubeflow-user-example-com"
+
+
+@dsl.component
+def add(a: float, b: float) -> float:
+    """Calculates sum of two arguments"""
+    return a + b
+
+
+@dsl.pipeline(
+    name="Addition pipeline",
+    description="An example pipeline that performs addition calculations.",
+)
+def add_pipeline(
+    a: float = 1.0,
+    b: float = 7.0,
+):
+    first_add_task = add(a=a, b=4.0)
+    add(a=first_add_task.output, b=b)
+
+
+try:
+    logger.info(
+        f"Trying to get experiment from {experiment_name=} {experiment_namespace=}."
+    )
+    experiment = client.get_experiment(
+        experiment_name=experiment_name, namespace=experiment_namespace
+    )
+    logger.info("Experiment found!")
+except Exception:
+    logger.info("Experiment not found, trying to create experiment.")
+    experiment = client.create_experiment(
+        name=experiment_name, namespace=experiment_namespace
+    )
+    logger.info("Experiment created!")
+
+try:
+    logger.info("Trying to create Pipeline Run.")
+    run = client.create_run_from_pipeline_func(
+        add_pipeline,
+        arguments={"a": 7.0, "b": 8.0},
+        experiment_id=experiment.experiment_id,
+        enable_caching=False,
+    )
+except Exception as e:
+    logger.error(
+        f"Failed to create Pipeline Run. Exception: {e.__class__.__name__}: {str(e)}"
+    )
+    raise SystemExit(1)
+
+while True:
+    live_run = client.get_run(run_id=run.run_id)
+    logger.info(f"Pipeline Run State: {live_run.state}.")
+
+    minutes_from_pipeline_run_start = (
+        datetime.now(timezone.utc) - live_run.created_at
+    ).seconds / 60
+
+    if minutes_from_pipeline_run_start > 5:
+        logger.debug(
+            "Pipeline is running for more than 5 minutes, "
+            f"showing pod states in {experiment_namespace=}."
+        )
+        subprocess.run(["kubectl", "get", "pods"])
+
+    if live_run.finished_at > live_run.created_at:
+        logger.info("Finished Pipeline Run!")
+        logger.info(
+            f"Pipeline was running for {minutes_from_pipeline_run_start:0.2} minutes."
+        )
+        logger.info(f"Pipeline Run finished in state: {live_run.state}.")
+        logger.info(f"Pipeline Run finished with error: {live_run.error}.")
+
+        if live_run.state != "SUCCEEDED":
+            logger.warn("The Pipeline Run finished but has failed...")
+
+            logger.warn("Running 'kubectl get pods':")
+            subprocess.run(["kubectl", "get", "pods"])
+
+            logger.warn("Running 'kubectl describe wf':")
+            subprocess.run(["kubectl", "describe", "wf"])
+
+            raise SystemExit(1)
+        break
+    else:
+        logger.info("Waiting for pipeline to finish...")
+    sleep(5)