end-to-end integration tests

.github/workflows/dex_oauth2-proxy_test.yaml

@@ -40,41 +40,26 @@ jobs:
       run: ./tests/gh-actions/install_multi_tenancy.sh
 
     - name: Install dex
-      run: |
-        echo "Installing Dex..."
-        kustomize build ./common/dex/overlays/oauth2-proxy | kubectl apply -f -
-
-        echo "Waiting for pods in auth namespace to become ready..."
-        kubectl wait --for=condition=Ready pods --all --timeout=180s -n auth
+      run: ./tests/gh-actions/install_dex.sh
 
     - name: Install central-dashboard
-      run: |
-        kustomize build apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f -
-        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s
+      run: ./tests/gh-actions/install_central_dashboard.sh
 
     - name: Create KF Profile
-      run: |
-        kustomize build common/user-namespace/base | kubectl apply -f -
-        sleep 30 # for the Profile controller to create the namespace from the profile
-        PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name')
-        if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then
-          echo "Error: profiles-deployment pod not found in kubeflow namespace."
-          exit 1
-        fi
-        kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
-        KF_PROFILE=kubeflow-user-example-com
-        kubectl -n $KF_PROFILE get pods,configmaps,secrets
-
-    - name: port forward
-      run: |
-        ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
-        nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 &
-        while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
+      run: ./tests/gh-actions/install_kubeflow_profile.sh
+
+    - name: Port forward the istio-ingress gateway
+      run: ./tests/gh-actions/port_forward_gateway.sh
 
-    - name: test dex login
+    - name: Test dex login
       run: |
-        pip3 install requests
-        ./tests/gh-actions/test_dex_login.py
+        # Install Python requirements
+        pip3 install -q requests
+
+        # Run the Dex login test
+        python3 tests/gh-actions/test_dex_login.py
+
+        echo "Dex login test completed successfully."
 
     - name: Apply Pod Security Standards baseline levels for static namespaces
       run: ./tests/gh-actions/enable_baseline_PSS.sh

.github/workflows/kserve_m2m_test.yaml

@@ -53,18 +53,7 @@ jobs:
       run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
 
     - name: Create KF Profile
-      run: |
-        kustomize build common/user-namespace/base | kubectl apply -f -
-        sleep 30 # for the Profile controller to create the namespace from the profile
-
-        PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name')
-        if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then
-          echo "Error: profiles-deployment pod not found in kubeflow namespace."
-          exit 1
-        fi
-        kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
-        KF_PROFILE=kubeflow-user-example-com
-        kubectl -n $KF_PROFILE get pods,configmaps,secrets
+      run: ./tests/gh-actions/install_kubeflow_profile.sh
 
     - name: Diagnose KServe Service Labels
       run: |
@@ -160,10 +149,7 @@ jobs:
       run: pip install -r ./apps/kserve/tests/requirements.txt
 
     - name: Port forward
-      run: |
-        INGRESS_GATEWAY_SERVICE=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
-        nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
-        while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
+      run: ./tests/gh-actions/port_forward_gateway.sh
 
     - name: Verify CNI is being used instead of init containers
       run: |

.github/workflows/pipeline_run_from_notebook.yaml

@@ -54,7 +54,7 @@ jobs:
           --field-selector=status.phase!=Succeeded
 
     - name: Create KF Profile
-      run: kustomize build common/user-namespace/base | kubectl apply -f -
+      run: ./tests/gh-actions/install_kubeflow_profile.sh
 
     - name: Apply PodDefaults to access ml-pipeline with projected token
       run: kubectl apply -f tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml

.github/workflows/pipeline_swfs_test.yaml

@@ -48,27 +48,11 @@ jobs:
       run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
 
     - name: Create KF Profile
-      run: |
-        kustomize build common/user-namespace/base | kubectl apply -f -
-        sleep 30 # for the metacontroller to create the secret
-
-        METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name')
-        if [[ -z "$METACONTROLLER_POD" ]]; then
-          echo "Error: metacontroller pod not found in kubeflow namespace."
-          exit 1
-        fi
-        kubectl logs -n kubeflow "$METACONTROLLER_POD"
-
-        PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name')
-        if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then
-          echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace."
-          exit 1
-        fi
-        kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD"
+      run: ./tests/gh-actions/install_kubeflow_profile.sh
 
+    - name: Verify Pipeline Integration
+      run: |
         KF_PROFILE=kubeflow-user-example-com
-        kubectl -n $KF_PROFILE get pods,configmaps,secrets
-
         if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then
           echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE"
           exit 1
@@ -81,11 +65,8 @@ jobs:
         kubectl -n kubeflow wait --for=condition=available --timeout=600s deploy/seaweedfs
         kubectl -n kubeflow exec deploy/seaweedfs -c seaweedfs -- sh -c "echo \"s3.configure -user minio -access_key minio -secret_key minio123 -actions Read,Write,List -apply\" | /usr/bin/weed shell"
 
-    - name: port forward
-      run: |
-        ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
-        nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 &
-        while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
+    - name: Port forward
+      run: ./tests/gh-actions/port_forward_gateway.sh
 
     - name: List and deploy test pipeline with authorized ServiceAccount Token
       run: |

.github/workflows/pipeline_test.yaml

@@ -49,38 +49,19 @@ jobs:
       run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
 
     - name: Create KF Profile
-      run: |
-        kustomize build common/user-namespace/base | kubectl apply -f -
-        sleep 30 # for the metacontroller to create the secret
-
-        METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name')
-        if [[ -z "$METACONTROLLER_POD" ]]; then
-          echo "Error: metacontroller pod not found in kubeflow namespace."
-          exit 1
-        fi
-        kubectl logs -n kubeflow "$METACONTROLLER_POD"
-
-        PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name')
-        if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then
-          echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace."
-          exit 1
-        fi
-        kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD"
+      run: ./tests/gh-actions/install_kubeflow_profile.sh
 
+    - name: Verify Pipeline Integration
+      run: |
         KF_PROFILE=kubeflow-user-example-com
-        kubectl -n $KF_PROFILE get pods,configmaps,secrets
-
         if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then
           echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE"
           exit 1
         fi
         kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' '
 
-    - name: port forward
-      run: |
-        ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
-        nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 &
-        while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
+    - name: Port forward
+      run: ./tests/gh-actions/port_forward_gateway.sh
 
     - name: List and deploy test pipeline with authorized ServiceAccount Token
       run: |

apps/kserve/tests/test_sklearn.py

@@ -43,7 +43,7 @@ def test_sklearn_kserve():
 
     isvc = V1beta1InferenceService(
         api_version=constants.KSERVE_V1BETA1,
-        kind=constants.KSERVE_KIND,
+        kind="InferenceService",
         metadata=client.V1ObjectMeta(
             name=service_name, namespace=KSERVE_TEST_NAMESPACE
         ),

tests/gh-actions/enable_baseline_PSS.sh

@@ -4,7 +4,12 @@ NAMESPACES=("istio-system" "auth" "cert-manager" "oauth2-proxy" "kubeflow")
 
 for NAMESPACE in "${NAMESPACES[@]}"; do
     if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
-        echo "Patching the PSS-baseline labels for namespace $NAMESPACE..."
-        kubectl patch namespace $NAMESPACE --patch-file ./experimental/security/PSS/static/baseline/patches/${NAMESPACE}-labels.yaml
+        # Check if the patch file exists
+        if [ -f "./experimental/security/PSS/static/baseline/patches/${NAMESPACE}-labels.yaml" ]; then
+            echo "Patching the PSS-baseline labels for namespace $NAMESPACE..."
+            kubectl patch namespace $NAMESPACE --patch-file ./experimental/security/PSS/static/baseline/patches/${NAMESPACE}-labels.yaml
+        else
+            echo "Patch file for namespace $NAMESPACE not found, skipping..."
+        fi
     fi
 done

tests/gh-actions/enable_restricted_PSS.sh

@@ -4,7 +4,12 @@ NAMESPACES=("istio-system" "auth" "cert-manager" "oauth2-proxy" "kubeflow")
 
 for NAMESPACE in "${NAMESPACES[@]}"; do
     if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
-        echo "Patching the PSS-restricted labels for namespace $NAMESPACE..."
-        kubectl patch namespace $NAMESPACE --patch-file ./experimental/security/PSS/static/restricted/patches/${NAMESPACE}-labels.yaml
+        # Check if the patch file exists
+        if [ -f "./experimental/security/PSS/static/restricted/patches/${NAMESPACE}-labels.yaml" ]; then
+            echo "Patching the PSS-restricted labels for namespace $NAMESPACE..."
+            kubectl patch namespace $NAMESPACE --patch-file ./experimental/security/PSS/static/restricted/patches/${NAMESPACE}-labels.yaml
+        else
+            echo "Patch file for namespace $NAMESPACE not found, skipping..."
+        fi
     fi
 done

tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh

@@ -54,8 +54,9 @@ nodes:
   image: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
 - role: worker
   image: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
-" | kind create cluster --config -
+" | kind create cluster --config - --wait 120s
 
+kubectl cluster-info
 
 echo "Install Kustomize ..."
 {

tests/gh-actions/install_central_dashboard.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+kustomize build apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f -
+kubectl wait --for=condition=Ready pods --all -n kubeflow --timeout=180s 

tests/gh-actions/install_dex.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euxo pipefail
+
+kustomize build ./common/dex/overlays/oauth2-proxy | kubectl apply -f -
+kubectl wait --for=condition=Ready pods --all --timeout=180s -n auth 

tests/gh-actions/install_knative.sh

@@ -17,6 +17,10 @@ set -e
 kustomize build common/istio-1-24/cluster-local-gateway/base | kubectl apply -f -
 kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
 
-kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s \
-  --field-selector=status.phase!=Succeeded
-kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s --field-selector=status.phase!=Succeeded
+kubectl wait --for=condition=Available deployment/activator -n knative-serving --timeout=10s
+kubectl wait --for=condition=Available deployment/autoscaler -n knative-serving --timeout=10s
+kubectl wait --for=condition=Available deployment/controller -n knative-serving --timeout=10s
+kubectl wait --for=condition=Available deployment/webhook -n knative-serving --timeout=10s
+kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving # TODO this will be moved into the manifests in https://github.com/kubeflow/manifests/pull/3084
+kubectl get deployment -n knative-serving

tests/gh-actions/install_kserve.sh

@@ -13,3 +13,10 @@ kustomize build kserve | kubectl apply --server-side --force-conflicts -f -
 kustomize build models-web-app/overlays/kubeflow | kubectl apply --server-side --force-conflicts -f -
 kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
   --field-selector=status.phase!=Succeeded
+kubectl wait --for=condition=Available deployment/kserve-controller-manager -n kubeflow --timeout=10s
+kubectl wait --for=condition=Available deployment/kserve-models-web-app -n kubeflow --timeout=10s
+kubectl get deployment -n kubeflow -l app.kubernetes.io/name=kserve
+kubectl get crd | grep -E 'inferenceservice|servingruntimes'
+
+# Return to the original directory
+cd ../../

tests/gh-actions/install_kubeflow_profile.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euxo
+kustomize build common/user-namespace/base | kubectl apply -f -
+sleep 30 # Let the profile controler reconcile the namespace
+PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name')
+kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
+KF_PROFILE=kubeflow-user-example-com
+kubectl -n $KF_PROFILE get pods,configmaps,secrets

tests/gh-actions/install_oauth2-proxy.sh

@@ -9,4 +9,5 @@ echo "Waiting for all oauth2-proxy pods to become ready..."
 kubectl wait --for=condition=Ready pod -l 'app.kubernetes.io/name=oauth2-proxy' --timeout=180s -n oauth2-proxy
 
 echo "Waiting for all cluster-jwks-proxy pods to become ready..."
-kubectl wait --for=condition=Ready pod -l 'app.kubernetes.io/name=cluster-jwks-proxy' --timeout=180s -n istio-system
+kubectl wait --for=condition=Ready pod -l 'app.kubernetes.io/name=cluster-jwks-proxy' --timeout=180s -n istio-system 
+kubectl wait --for=condition=Available deployment -n oauth2-proxy oauth2-proxy --timeout=180s

tests/gh-actions/install_pipelines.sh

@@ -9,4 +9,15 @@ kustomize build env/cert-manager/platform-agnostic-multi-user | kubectl apply -f
 sleep 60
 kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
   --field-selector=status.phase!=Succeeded
+
+kubectl wait --for=condition=Available deployment/ml-pipeline -n kubeflow --timeout=10s 
+kubectl wait --for=condition=Available deployment/ml-pipeline-ui -n kubeflow --timeout=10s 
+kubectl wait --for=condition=Available deployment/ml-pipeline-persistenceagent -n kubeflow --timeout=10s
+kubectl wait --for=condition=Available deployment/ml-pipeline-scheduledworkflow -n kubeflow --timeout=10s
+kubectl wait --for=condition=Available deployment/ml-pipeline-viewer-crd -n kubeflow --timeout=10s 
+kubectl wait --for=condition=Available deployment/cache-server -n kubeflow --timeout=10s 
+kubectl wait --for=condition=Available deployment/metadata-writer -n kubeflow --timeout=10s 
+kubectl wait --for=condition=Available deployment/minio -n kubeflow --timeout=10s
+kubectl wait --for=condition=Available deployment/mysql -n kubeflow --timeout=10s
+kubectl get deployment -n kubeflow -l app=ml-pipeline
 cd -

tests/gh-actions/install_training_operator.sh

@@ -6,4 +6,7 @@ cd apps/training-operator/upstream
 kustomize build overlays/kubeflow | kubectl apply --server-side --force-conflicts -f -
 kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
   --field-selector=status.phase!=Succeeded
+kubectl wait --for=condition=Available deployment/training-operator -n kubeflow --timeout=10s
+kubectl get crd | grep -E 'tfjobs.kubeflow.org|pytorchjobs.kubeflow.org'
+kubectl get deployment -n kubeflow training-operator
 cd -