From 5821dd19a9fd40d22b16668a6f83195397d18c73 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Thu, 2 Oct 2025 19:56:45 +0530 Subject: [PATCH 01/14] Fix:consolidate kserve test workflows into job Signed-off-by: madmecodes --- .github/workflows/kserve_test.yaml | 159 ++--------------------------- tests/kserve_test.sh | 5 +- 2 files changed, 10 insertions(+), 154 deletions(-) diff --git a/.github/workflows/kserve_test.yaml b/.github/workflows/kserve_test.yaml index ff652ba352..dbdd4aedfe 100644 --- a/.github/workflows/kserve_test.yaml +++ b/.github/workflows/kserve_test.yaml @@ -18,14 +18,13 @@ on: - common/knative/** - tests/knative_install.sh - tests/*authentication*test.sh - - tests/final_validation.sh permissions: contents: read actions: read jobs: - test-basic-kserve: + test-kserve: runs-on: ubuntu-latest steps: - name: Checkout @@ -72,162 +71,16 @@ jobs: - name: Port forward run: ./tests/port_forward_gateway.sh - - name: Run KServe tests - run: ./tests/kserve_test.sh kubeflow-user-example-com - - - name: Detailed KServe Access Diagnostics - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)" - - echo "=== AuthorizationPolicy Details ===" - kubectl get authorizationpolicy -n kubeflow-user-example-com -o yaml - - echo "=== Detailed Curl Test ===" - curl -vv \ - -H "Host: isvc-sklearn.kubeflow-user-example-com.example.com" \ - -H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \ - -H "Content-Type: application/json" \ - "http://${KSERVE_INGRESS_HOST_PORT}/v1/models/isvc-sklearn:predict" \ - -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}' - - - name: Run kserve models webapp test - run: | - kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app - - - name: Apply Pod Security Standards restricted levels - run: ./tests/PSS_enable.sh - - test-jwt-authentication: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - - name: Install kubectl - run: ./tests/kubectl_install.sh - - - name: Create kubeflow namespace - run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - - name: Install Istio CNI - run: ./tests/istio-cni_install.sh - - - name: Install oauth2-proxy - run: ./tests/oauth2-proxy_install.sh - - - name: Install knative CNI with secure cluster-local-gateway - run: ./tests/knative_install.sh - - - name: Verify secure cluster-local-gateway configuration - run: | - kubectl get authorizationpolicy,requestauthentication -n istio-system | grep cluster-local-gateway - kubectl get requestauthentication cluster-local-gateway-jwt -n istio-system -o yaml - kubectl get authorizationpolicy cluster-local-gateway -n istio-system -o yaml - kubectl get authorizationpolicy cluster-local-gateway-require-jwt -n istio-system -o yaml - - - name: Setup python 3.12 - uses: actions/setup-python@v4 - with: - python-version: 3.12 - - - name: Port forward - run: ./tests/port_forward_gateway.sh - - - name: Wait for cluster-local-gateway to be ready + - name: Wait for Istio configurations to propagate run: | kubectl wait --for=condition=Available --timeout=120s deployment/cluster-local-gateway -n istio-system - sleep 100 + sleep 60 - - name: Run Basic JWT Authentication Tests - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - curl -s -o /dev/null -w "%{http_code}" -H "Host: test.example.com" "http://localhost:8080/" | grep -q "403" + - name: Run KServe tests + run: ./tests/kserve_test.sh kubeflow-user-example-com - name: Run Knative Service JWT Authentication Tests - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - ./tests/knative_authentication_test.sh - - - name: Test External Access Configuration - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - ./tests/kserve_setup_external_access.sh kubeflow-user-example-com secure-model-predictor - # Test external access pattern - TOKEN=$(kubectl -n kubeflow-user-example-com create token default-editor) - RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - "http://localhost:8080/kserve/kubeflow-user-example-com/secure-model-predictor/" \ - 2>/dev/null || echo "404") - if [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "503" ]; then - exit 1 - fi - - - name: Apply Pod Security Standards restricted levels - run: ./tests/PSS_enable.sh - - test-secure-authentication: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - - name: Install kubectl - run: ./tests/kubectl_install.sh - - - name: Create kubeflow namespace - run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - - name: Install Istio CNI - run: ./tests/istio-cni_install.sh - - - name: Install oauth2-proxy - run: ./tests/oauth2-proxy_install.sh - - - name: Install cert-manager - run: ./tests/cert_manager_install.sh - - - name: Install knative CNI (with secure cluster-local-gateway) - run: ./tests/knative_install.sh - - - name: Install KServe - run: ./tests/kserve_install.sh - - - name: Install KF Multi Tenancy - run: ./tests/multi_tenancy_install.sh - - - name: Install kubeflow-istio-resources - run: kustomize build common/istio/kubeflow-istio-resources/base | kubectl apply -f - - - - name: Create KF Profile - run: ./tests/kubeflow_profile_install.sh - - - name: Setup python 3.12 - uses: actions/setup-python@v4 - with: - python-version: 3.12 - - - name: Port forward - run: ./tests/port_forward_gateway.sh - - - name: Verify JWT authentication policies are applied - run: | - kubectl get authorizationpolicy cluster-local-gateway-require-jwt -n istio-system - kubectl get requestauthentication cluster-local-gateway-jwt -n istio-system - kubectl get authorizationpolicy cluster-local-gateway -n istio-system - kubectl get deployment cluster-local-gateway -n istio-system - kubectl wait --for=condition=Available deployment/cluster-local-gateway -n istio-system --timeout=120s - kubectl get pods -n istio-system -l app=cluster-jwks-proxy | grep -q Running || kubectl get pods -n istio-system -l app=cluster-jwks-proxy - - - name: Wait for configurations to propagate - run: sleep 60 + run: ./tests/knative_authentication_test.sh - name: Run KServe secure authentication tests run: ./tests/kserve_jwt_authentication_test.sh kubeflow-user-example-com diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 2630a3f5ba..52904ed569 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -130,4 +130,7 @@ kubectl delete inferenceservice isvc-sklearn -n ${NAMESPACE} --ignore-not-found= if cd ${TEST_DIRECTORY}; then pytest . -vs --log-level info || true -fi \ No newline at end of file +fi + +# Test KServe models web app deployment +kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app \ No newline at end of file From 04e50c199cc1b39370dde8e57435b855395b765b Mon Sep 17 00:00:00 2001 From: madmecodes Date: Thu, 2 Oct 2025 20:10:35 +0530 Subject: [PATCH 02/14] update end to end test for kserve and knative tests Signed-off-by: madmecodes --- .github/workflows/full_kubeflow_integration_test.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/full_kubeflow_integration_test.yaml b/.github/workflows/full_kubeflow_integration_test.yaml index 61d13d6463..d6fd7fd048 100644 --- a/.github/workflows/full_kubeflow_integration_test.yaml +++ b/.github/workflows/full_kubeflow_integration_test.yaml @@ -195,8 +195,13 @@ jobs: run: ./tests/katib_test.sh "${KF_PROFILE}" - name: Run KServe Test - run: | - ./tests/kserve_test.sh ${KF_PROFILE} + run: ./tests/kserve_test.sh ${KF_PROFILE} + + - name: Run Knative Service JWT Authentication Tests + run: ./tests/knative_authentication_test.sh + + - name: Run KServe secure authentication tests + run: ./tests/kserve_jwt_authentication_test.sh ${KF_PROFILE} - name: Run Spark Test run: chmod u+x tests/*.sh && ./tests/spark_test.sh "${KF_PROFILE}" From ac486bcdd5fa5779b7c5c12bb0126ae29ea381e5 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Thu, 2 Oct 2025 20:20:14 +0530 Subject: [PATCH 03/14] fix: not just ready must be functional Signed-off-by: madmecodes --- tests/kserve_jwt_authentication_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kserve_jwt_authentication_test.sh b/tests/kserve_jwt_authentication_test.sh index 69b847f57c..28614b7571 100755 --- a/tests/kserve_jwt_authentication_test.sh +++ b/tests/kserve_jwt_authentication_test.sh @@ -41,7 +41,10 @@ EOF then echo "InferenceService created successfully, waiting for it to be ready..." kubectl wait --for=condition=Ready inferenceservice/test-sklearn-secure -n ${NAMESPACE} --timeout=180s || echo "InferenceService not ready, continuing with JWT tests..." - + + # Wait for Istio configurations to propagate + sleep 60 + # Create AuthorizationPolicy to allow authenticated access cat < Date: Thu, 2 Oct 2025 20:41:10 +0530 Subject: [PATCH 04/14] fix: sleep after deleting Signed-off-by: madmecodes --- tests/kserve_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 52904ed569..1f9f362574 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -126,7 +126,9 @@ spec: serving.knative.dev/service: isvc-sklearn-predictor EOF +# Delete isvc-sklearn before running pytest (pytest will recreate it) kubectl delete inferenceservice isvc-sklearn -n ${NAMESPACE} --ignore-not-found=true +sleep 10 if cd ${TEST_DIRECTORY}; then pytest . -vs --log-level info || true From 93f1dc004cd8009328fd72358d573204e4252576 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Thu, 2 Oct 2025 20:57:22 +0530 Subject: [PATCH 05/14] fix: wait for deletion Signed-off-by: madmecodes --- tests/kserve_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 1f9f362574..1b763b0c5e 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -128,7 +128,8 @@ EOF # Delete isvc-sklearn before running pytest (pytest will recreate it) kubectl delete inferenceservice isvc-sklearn -n ${NAMESPACE} --ignore-not-found=true -sleep 10 +# Wait for deletion to complete +kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=60s || true if cd ${TEST_DIRECTORY}; then pytest . -vs --log-level info || true From 36fff35b900bc1d3002579690af500bf4f585a77 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Fri, 3 Oct 2025 19:28:57 +0530 Subject: [PATCH 06/14] remove dublicate run of pytest, now it runs once in kserve_test only Signed-off-by: madmecodes --- tests/kserve_jwt_authentication_test.sh | 10 +--------- tests/kserve_test.sh | 7 ++++++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/kserve_jwt_authentication_test.sh b/tests/kserve_jwt_authentication_test.sh index 28614b7571..325eb89630 100755 --- a/tests/kserve_jwt_authentication_test.sh +++ b/tests/kserve_jwt_authentication_test.sh @@ -2,8 +2,6 @@ set -euxo pipefail NAMESPACE=${1:-kubeflow-user-example-com} -SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -TEST_DIRECTORY="${SCRIPT_DIRECTORY}/kserve" if ! command -v pytest &> /dev/null; then true @@ -110,10 +108,4 @@ if [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "503 fi # Clean up -kubectl delete namespace attacker-namespace --ignore-not-found=true - - -# Run existing pytest tests if available -if command -v pytest &> /dev/null && [ -d "${TEST_DIRECTORY}" ]; then - cd ${TEST_DIRECTORY} && pytest . -vs --log-level info -fi \ No newline at end of file +kubectl delete namespace attacker-namespace --ignore-not-found=true \ No newline at end of file diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 1b763b0c5e..f8e12c91d9 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -128,8 +128,13 @@ EOF # Delete isvc-sklearn before running pytest (pytest will recreate it) kubectl delete inferenceservice isvc-sklearn -n ${NAMESPACE} --ignore-not-found=true + # Wait for deletion to complete -kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=60s || true +if kubectl get inferenceservice isvc-sklearn -n ${NAMESPACE} &>/dev/null; then + kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=120s +else + echo "isvc-sklearn does not exist, no need to wait for deletion" +fi if cd ${TEST_DIRECTORY}; then pytest . -vs --log-level info || true From 27b40cde85523342cd1dfbd9c95385a0bc14e137 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Fri, 3 Oct 2025 19:59:16 +0530 Subject: [PATCH 07/14] update: remove debug echo Signed-off-by: madmecodes --- tests/kserve_test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index f8e12c91d9..0420eee59b 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -132,8 +132,6 @@ kubectl delete inferenceservice isvc-sklearn -n ${NAMESPACE} --ignore-not-found= # Wait for deletion to complete if kubectl get inferenceservice isvc-sklearn -n ${NAMESPACE} &>/dev/null; then kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=120s -else - echo "isvc-sklearn does not exist, no need to wait for deletion" fi if cd ${TEST_DIRECTORY}; then From 784933d61be41e772a2bf906d9adeec2c6669685 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Fri, 10 Oct 2025 16:16:45 +0530 Subject: [PATCH 08/14] merge all the kserve tests into one file Signed-off-by: madmecodes --- .../full_kubeflow_integration_test.yaml | 6 - .github/workflows/kserve_test.yaml | 8 - tests/knative_authentication_test.sh | 116 ----------- tests/kserve_jwt_authentication_test.sh | 111 ---------- tests/kserve_test.sh | 195 +++++++++++++++++- 5 files changed, 189 insertions(+), 247 deletions(-) delete mode 100755 tests/knative_authentication_test.sh delete mode 100755 tests/kserve_jwt_authentication_test.sh diff --git a/.github/workflows/full_kubeflow_integration_test.yaml b/.github/workflows/full_kubeflow_integration_test.yaml index d6fd7fd048..d95f7c419c 100644 --- a/.github/workflows/full_kubeflow_integration_test.yaml +++ b/.github/workflows/full_kubeflow_integration_test.yaml @@ -197,12 +197,6 @@ jobs: - name: Run KServe Test run: ./tests/kserve_test.sh ${KF_PROFILE} - - name: Run Knative Service JWT Authentication Tests - run: ./tests/knative_authentication_test.sh - - - name: Run KServe secure authentication tests - run: ./tests/kserve_jwt_authentication_test.sh ${KF_PROFILE} - - name: Run Spark Test run: chmod u+x tests/*.sh && ./tests/spark_test.sh "${KF_PROFILE}" diff --git a/.github/workflows/kserve_test.yaml b/.github/workflows/kserve_test.yaml index dbdd4aedfe..b615114a27 100644 --- a/.github/workflows/kserve_test.yaml +++ b/.github/workflows/kserve_test.yaml @@ -9,7 +9,6 @@ on: - tests/kserve/** - tests/kserve_test.sh - tests/kserve_install.sh - - tests/kserve_jwt_authentication_test.sh - common/istio*/** - common/oauth2-proxy/** - tests/oauth2-proxy_install.sh @@ -17,7 +16,6 @@ on: - tests/istio* - common/knative/** - tests/knative_install.sh - - tests/*authentication*test.sh permissions: contents: read @@ -79,11 +77,5 @@ jobs: - name: Run KServe tests run: ./tests/kserve_test.sh kubeflow-user-example-com - - name: Run Knative Service JWT Authentication Tests - run: ./tests/knative_authentication_test.sh - - - name: Run KServe secure authentication tests - run: ./tests/kserve_jwt_authentication_test.sh kubeflow-user-example-com - - name: Apply Pod Security Standards restricted levels run: ./tests/PSS_enable.sh diff --git a/tests/knative_authentication_test.sh b/tests/knative_authentication_test.sh deleted file mode 100755 index 01359d66e8..0000000000 --- a/tests/knative_authentication_test.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -set -euo pipefail - -PRIMARY_NAMESPACE="kubeflow-user-example-com" -ATTACKER_NAMESPACE="kubeflow-user-attacker" -INGRESS_HOST_PORT="${KSERVE_INGRESS_HOST_PORT:-localhost:8080}" - -function setup_test_environment() { - kubectl create namespace $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - - kubectl create namespace $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - - kubectl create serviceaccount default-editor -n $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - - kubectl create serviceaccount attacker-sa -n $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - - - cat < /dev/null - - RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \ - "http://localhost:8081/") - - if [ "$RESPONSE" != "403" ]; then - exit 1 - fi - - kill $PF_PID 2>/dev/null || true -} - -function cleanup() { - kubectl delete namespace $ATTACKER_NAMESPACE --ignore-not-found=true - kubectl delete ksvc secure-model-predictor -n $PRIMARY_NAMESPACE --ignore-not-found=true -} - -function main() { - setup_test_environment - test_gateway_authentication - test_namespace_isolation - test_cluster_local_gateway - cleanup -} - -main \ No newline at end of file diff --git a/tests/kserve_jwt_authentication_test.sh b/tests/kserve_jwt_authentication_test.sh deleted file mode 100755 index 325eb89630..0000000000 --- a/tests/kserve_jwt_authentication_test.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -NAMESPACE=${1:-kubeflow-user-example-com} - -if ! command -v pytest &> /dev/null; then - true -fi - -export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080} -export KSERVE_TEST_NAMESPACE=${NAMESPACE} - -# Create service account if it doesn't exist -kubectl create serviceaccount default-editor -n ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - - -# Create JWT token -export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)" - -# Try to deploy test InferenceService (may fail if KServe webhooks not ready) -set +e -if cat </dev/null; then kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=120s fi @@ -138,5 +134,192 @@ if cd ${TEST_DIRECTORY}; then pytest . -vs --log-level info || true fi -# Test KServe models web app deployment -kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app \ No newline at end of file +kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app + +PRIMARY_NAMESPACE="kubeflow-user-example-com" +ATTACKER_NAMESPACE="kubeflow-user-attacker" +INGRESS_HOST_PORT="${KSERVE_INGRESS_HOST_PORT:-localhost:8080}" + +kubectl create namespace $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - +kubectl create namespace $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - +kubectl create serviceaccount default-editor -n $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - +kubectl create serviceaccount attacker-sa -n $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - + +cat < /dev/null + +RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \ + "http://localhost:8081/") + +if [ "$RESPONSE" != "403" ]; then + exit 1 +fi + +kill $PF_PID 2>/dev/null || true + +kubectl delete namespace $ATTACKER_NAMESPACE --ignore-not-found=true +kubectl delete ksvc secure-model-predictor -n $PRIMARY_NAMESPACE --ignore-not-found=true + +kubectl create serviceaccount default-editor -n ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - + +export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)" + +set +e +if cat < Date: Fri, 10 Oct 2025 16:31:51 +0530 Subject: [PATCH 09/14] remove service account creation debug line already created Signed-off-by: madmecodes --- tests/kserve_test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index f95dfe5dde..3a1b5f3d70 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -142,7 +142,6 @@ INGRESS_HOST_PORT="${KSERVE_INGRESS_HOST_PORT:-localhost:8080}" kubectl create namespace $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - kubectl create namespace $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - -kubectl create serviceaccount default-editor -n $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - kubectl create serviceaccount attacker-sa -n $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - cat < Date: Fri, 10 Oct 2025 17:14:23 +0530 Subject: [PATCH 10/14] update: remove redundant tests policies Signed-off-by: madmecodes --- tests/kserve_test.sh | 159 +++++-------------------------------------- 1 file changed, 18 insertions(+), 141 deletions(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 3a1b5f3d70..bd1edc907d 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -13,6 +13,7 @@ export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080} export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)" export KSERVE_TEST_NAMESPACE=${NAMESPACE} +# Path-based routing cat < /dev/null RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \ + -H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \ "http://localhost:8081/") if [ "$RESPONSE" != "403" ]; then @@ -226,99 +198,4 @@ fi kill $PF_PID 2>/dev/null || true -kubectl delete namespace $ATTACKER_NAMESPACE --ignore-not-found=true -kubectl delete ksvc secure-model-predictor -n $PRIMARY_NAMESPACE --ignore-not-found=true - -kubectl create serviceaccount default-editor -n ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - - -export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)" - -set +e -if cat < Date: Sun, 19 Oct 2025 11:41:16 +0530 Subject: [PATCH 11/14] test: attacker namespace added Signed-off-by: madmecodes --- tests/kserve_test.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index bd1edc907d..c61425189c 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -196,6 +196,30 @@ if [ "$RESPONSE" != "403" ]; then exit 1 fi +# Test namespace isolation - attacker in different namespace should not have access +ATTACKER_NAMESPACE="attacker-namespace" +kubectl create namespace ${ATTACKER_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - + +cat </dev/null || true +kubectl delete namespace ${ATTACKER_NAMESPACE} --ignore-not-found=true kubectl delete ksvc secure-model-predictor -n ${NAMESPACE} --ignore-not-found=true From 5957fee52e0024f89265b7f5658c0376de4ac3c6 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Fri, 31 Oct 2025 22:52:12 +0530 Subject: [PATCH 12/14] test: replicating prev test file Signed-off-by: madmecodes --- tests/kserve_test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index c61425189c..99af9ca92d 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -127,6 +127,8 @@ if kubectl get inferenceservice isvc-sklearn -n ${NAMESPACE} &>/dev/null; then kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=120s fi +sleep 5 + if cd ${TEST_DIRECTORY}; then pytest . -vs --log-level info || true fi @@ -215,7 +217,9 @@ RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ -H "Authorization: Bearer $ATTACKER_TOKEN" \ "http://localhost:8081/") -if [ "$RESPONSE" != "403" ]; then +# The attacker token can reach the service through the cluster-local-gateway, +# but may get 404 or 503 depending on whether the request is fully processed +if [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "503" ]; then exit 1 fi From 65cba403260dbb95a26c5185125fc5b4f6dab720 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Sat, 1 Nov 2025 11:49:36 +0530 Subject: [PATCH 13/14] update: attacker ns must not have access Signed-off-by: madmecodes --- tests/kserve_test.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 99af9ca92d..3e709ab83f 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -198,7 +198,7 @@ if [ "$RESPONSE" != "403" ]; then exit 1 fi -# Test namespace isolation - attacker in different namespace should not have access +# Test namespace isolation - attacker in different namespace should NOT have access ATTACKER_NAMESPACE="attacker-namespace" kubectl create namespace ${ATTACKER_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - @@ -210,6 +210,18 @@ metadata: namespace: ${ATTACKER_NAMESPACE} EOF +# Test 1: Unauthenticated request from attacker namespace should be REJECTED +RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \ + "http://localhost:8081/") + +if [ "$RESPONSE" == "200" ]; then + echo "FAIL: Unauthenticated attacker namespace request should be rejected, got $RESPONSE" + exit 1 +fi + +# Test 2: Authenticated request from attacker namespace should ALSO be REJECTED +# Enforces strict namespace isolation - even with a valid token from another namespace ATTACKER_TOKEN=$(kubectl -n ${ATTACKER_NAMESPACE} create token attacker-sa) RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ @@ -217,9 +229,8 @@ RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ -H "Authorization: Bearer $ATTACKER_TOKEN" \ "http://localhost:8081/") -# The attacker token can reach the service through the cluster-local-gateway, -# but may get 404 or 503 depending on whether the request is fully processed -if [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "503" ]; then +if [ "$RESPONSE" == "200" ]; then + echo "FAIL: Attacker namespace token should be rejected (namespace isolation), got $RESPONSE" exit 1 fi From 0fc5f9e3ac61545c95c2551db3a20c9fa9f4dbc9 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Sun, 16 Nov 2025 18:29:06 +0530 Subject: [PATCH 14/14] update: remove duplicate test-sklearn service and simplify test structure Signed-off-by: madmecodes --- tests/kserve_test.sh | 90 +++++++++++--------------------------------- 1 file changed, 21 insertions(+), 69 deletions(-) diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh index 3e709ab83f..e1007b89d1 100755 --- a/tests/kserve_test.sh +++ b/tests/kserve_test.sh @@ -13,12 +13,17 @@ export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080} export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)" export KSERVE_TEST_NAMESPACE=${NAMESPACE} -# Path-based routing +# Test 1: Model Inference via KServe SDK (pytest creates isvc-sklearn internally) +if cd ${TEST_DIRECTORY}; then + pytest . -vs --log-level info || true +fi + +# Test 2: Path-based Routing & Ingress Gateway Security (VirtualService + AuthorizationPolicy) cat </dev/null; then - kubectl wait --for=delete inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=120s -fi - -sleep 5 - -if cd ${TEST_DIRECTORY}; then - pytest . -vs --log-level info || true -fi - kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app # Knative Service authentication via cluster-local-gateway @@ -178,16 +133,14 @@ if [ "$RESPONSE" != "401" ] && [ "$RESPONSE" != "403" ]; then exit 1 fi -# Verify cluster-local-gateway requires authentication +# Test 3: Cluster-local-gateway requires authentication kubectl port-forward -n istio-system svc/cluster-local-gateway 8081:80 & PF_PID=$! sleep 5 -PRIMARY_TOKEN=$(kubectl -n ${NAMESPACE} create token default-editor) - curl -s -o /dev/null -w "%{http_code}" \ -H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \ - -H "Authorization: Bearer $PRIMARY_TOKEN" \ + -H "Authorization: Bearer $KSERVE_M2M_TOKEN" \ "http://localhost:8081/" > /dev/null RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ @@ -198,7 +151,7 @@ if [ "$RESPONSE" != "403" ]; then exit 1 fi -# Test namespace isolation - attacker in different namespace should NOT have access +# Test 4: Namespace isolation - attacker in different namespace should NOT have access ATTACKER_NAMESPACE="attacker-namespace" kubectl create namespace ${ATTACKER_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - @@ -206,11 +159,11 @@ cat <