Add SeaweedFS namespace isolation security test to CI

.github/workflows/pipeline_swfs_test.yaml

@@ -6,23 +6,36 @@ on:
     # - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
     - .github/workflows/pipeline_swfs_test.yaml
     - apps/pipeline/upstream/**
-    # - tests/gh-actions/install_istio.sh
-    # - tests/gh-actions/install_cert_manager.sh
-    # - tests/gh-actions/install_oauth2-proxy.sh
-    # - common/cert-manager/**
-    # - common/oauth2-proxy/**
-    # - common/istio*/**
+    - tests/gh-actions/install_istio*.sh
+    - tests/gh-actions/install_cert_manager.sh
+    - tests/gh-actions/install_oauth2-proxy.sh
+    - common/cert-manager/**
+    - common/oauth2-proxy/**
+    - common/istio*/**
     - experimental/seaweedfs/**
+    - tests/gh-actions/test_swfs_namespace_isolation.sh
+    - tests/gh-actions/s3_test_helper.py
 
 jobs:
   build:
     timeout-minutes: 15
     runs-on:
-      labels: ubuntu-latest-16-cores
+      labels: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
 
+    - name: Remove unused software
+      run: |
+        df -h # Check disk space before removal
+        sudo rm -rf /usr/share/dotnet # Example: Remove .NET SDK
+        sudo rm -rf /usr/local/lib/android # Example: Remove Android SDK
+        sudo rm -rf /opt/ghc # Example: Remove Haskell
+        df -h # Check disk space after removal
+
+    - name: Proactively prune OCI system on GHA runner
+      run: docker system prune -a --volumes --force
+
     - name: Install KinD, Create KinD cluster and Install kustomize
       run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
 
@@ -62,6 +75,7 @@ jobs:
         fi
         kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' '
 
+
     - name: Port forward
       run: ./tests/gh-actions/port_forward_gateway.sh
 
@@ -72,6 +86,9 @@ jobs:
         TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
         python3 tests/gh-actions/test_pipeline_v1.py "${TOKEN}" "${KF_PROFILE}"
 
+    - name: Prune images inside Kind cluster
+      run: docker exec kind-control-plane bash -c "crictl images prune"
+
     - name: List and deploy test pipeline with V2 API
       run: |
         pip3 install kfp==2.13.0
@@ -87,6 +104,8 @@ jobs:
         python3 tests/gh-actions/test_pipeline_v2.py test_unauthorized_access "${TOKEN}" "${KF_PROFILE}"
         echo "Test succeeded. Token from unauthorized ServiceAccount cannot list pipelines in $KF_PROFILE namespace."
 
+    - name: Test SeaweedFS Namespace Isolation
+      run: ./tests/gh-actions/test_swfs_namespace_isolation.sh
 
     - name: Apply Pod Security Standards baseline levels for static namespaces
       run: ./tests/gh-actions/enable_baseline_PSS.sh
@@ -109,11 +128,13 @@ jobs:
         mkdir -p logs
         kubectl get all --all-namespaces > logs/resources.txt
         kubectl get events --all-namespaces --sort-by=.metadata.creationTimestamp > logs/events.txt
-        for namespace in kubeflow istio-system cert-manager auth kubeflow-user-example-com; do
-            kubectl describe pods -n $namespace > logs/$namespace-pods.txt
-            for pod in $(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}'); do
-                kubectl logs -n $namespace $pod --tail=100 > logs/$namespace-$pod.txt 2>&1 || true
-            done
+        for namespace in kubeflow istio-system cert-manager auth kubeflow-user-example-com test-profile-1 test-profile-2; do
+            if kubectl get namespace $namespace >/dev/null 2>&1; then
+                kubectl describe pods -n $namespace > logs/$namespace-pods.txt
+                for pod in $(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}'); do
+                    kubectl logs -n $namespace $pod --tail=100 > logs/$namespace-$pod.txt 2>&1 || true
+                done
+            fi
         done
 
     - name: Upload Diagnostic Logs

experimental/seaweedfs/base/seaweedfs/seaweedfs-pvc.yaml

@@ -8,4 +8,4 @@ spec:
   - ReadWriteOnce
   resources:
     requests:
-      storage: 20Gi
+      storage: 5Gi

tests/gh-actions/s3_test_helper.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""
+S3 helper script for SeaweedFS namespace isolation testing.
+Uses boto3 to perform S3 operations for security testing.
+"""
+
+import sys
+import boto3
+from botocore.exceptions import ClientError, NoCredentialsError
+import argparse
+
+
+def create_s3_client(access_key, secret_key, endpoint_url):
+    """Create S3 client with given credentials."""
+    return boto3.client(
+        's3',
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        endpoint_url=endpoint_url,
+        region_name='us-east-1'  # Required but not used by SeaweedFS
+    )
+
+
+def upload_file(access_key, secret_key, endpoint_url, bucket, key, content):
+    """Upload a file to S3."""
+    try:
+        s3_client = create_s3_client(access_key, secret_key, endpoint_url)
+        s3_client.put_object(
+            Bucket=bucket,
+            Key=key,
+            Body=content.encode('utf-8')
+        )
+        print(f"✓ Successfully uploaded file to s3://{bucket}/{key}")
+        return True
+    except Exception as e:
+        print(f"✗ Failed to upload file: {e}")
+        return False
+
+
+def download_file(access_key, secret_key, endpoint_url, bucket, key):
+    """Download a file from S3."""
+    try:
+        s3_client = create_s3_client(access_key, secret_key, endpoint_url)
+        response = s3_client.get_object(Bucket=bucket, Key=key)
+        content = response['Body'].read().decode('utf-8')
+        print(f"✓ Successfully downloaded file from s3://{bucket}/{key}")
+        print(f"File content: {content}")
+        return True, content
+    except ClientError as e:
+        error_code = e.response['Error']['Code']
+        if error_code in ['NoSuchKey', 'AccessDenied', 'Forbidden']:
+            print(f"✓ Access denied as expected: {error_code}")
+            return False, None
+        else:
+            print(f"✗ Unexpected error: {e}")
+            return False, None
+    except Exception as e:
+        print(f"✗ Failed to download file: {e}")
+        return False, None
+
+
+def main():
+    parser = argparse.ArgumentParser(description='S3 operations for SeaweedFS testing')
+    parser.add_argument('operation', choices=['upload', 'download'], help='Operation to perform')
+    parser.add_argument('--access-key', required=True, help='AWS access key')
+    parser.add_argument('--secret-key', required=True, help='AWS secret key')
+    parser.add_argument('--endpoint-url', required=True, help='S3 endpoint URL')
+    parser.add_argument('--bucket', required=True, help='S3 bucket name')
+    parser.add_argument('--key', required=True, help='S3 object key')
+    parser.add_argument('--content', help='Content to upload (for upload operation)')
+
+    args = parser.parse_args()
+
+    if args.operation == 'upload':
+        if not args.content:
+            print("Error: --content is required for upload operation")
+            sys.exit(1)
+        success = upload_file(args.access_key, args.secret_key, args.endpoint_url, 
+                             args.bucket, args.key, args.content)
+        sys.exit(0 if success else 1)
+
+    elif args.operation == 'download':
+        success, content = download_file(args.access_key, args.secret_key, args.endpoint_url,
+                                        args.bucket, args.key)
+        # For security test: success=True means unauthorized access (bad)
+        # success=False means access denied (good)
+        if args.key.startswith('private-artifacts/') and '/' in args.key[18:]:
+            # This is a cross-namespace access attempt
+            sys.exit(1 if success else 0)
+        else:
+            sys.exit(0 if success else 1)
+
+
+if __name__ == '__main__':
+    main()

tests/gh-actions/test_swfs_namespace_isolation.sh

@@ -0,0 +1,197 @@
+#!/bin/bash
+set -euxo pipefail
+
+echo "SeaweedFS Security Test - Unauthorized Access Check"
+echo "Testing if one namespace can access files from another namespace"
+
+# Check dependencies
+for cmd in kubectl python3; do
+    if ! command -v $cmd &> /dev/null; then
+        echo "Error: $cmd is required but not installed"
+        exit 1
+    fi
+done
+
+# Install boto3 if not available
+if ! python3 -c "import boto3" 2>/dev/null; then
+    echo "Installing boto3..."
+    pip3 install boto3
+fi
+
+PORT_FORWARD_PID=""
+# Cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    if [ -n "$PORT_FORWARD_PID" ]; then
+        kill $PORT_FORWARD_PID 2>/dev/null || true
+    fi
+    rm -f test-file.txt accessed-file.txt
+    kubectl delete profile test-profile-1 test-profile-2 --ignore-not-found
+}
+trap cleanup EXIT
+
+# Create test profiles
+create_profiles() {
+    echo "Creating test profiles..."
+
+    # Create both profiles
+    kubectl apply -f - <<EOF
+apiVersion: kubeflow.org/v1
+kind: Profile
+metadata:
+  name: test-profile-1
+spec:
+  owner:
+    kind: User
+    name: test-user-1@example.com
+---
+apiVersion: kubeflow.org/v1
+kind: Profile
+metadata:
+  name: test-profile-2
+spec:
+  owner:
+    kind: User
+    name: test-user-2@example.com
+EOF
+
+    # Wait for namespaces
+    echo "Waiting for namespaces..."
+    for i in {1..6}; do
+        if kubectl get namespace test-profile-1 test-profile-2 >/dev/null 2>&1; then
+            echo "Namespaces created"
+            return 0
+        fi
+        sleep 10
+    done
+
+    echo "Error: Namespaces not created"
+    exit 1
+}
+
+# Wait for S3 credentials
+wait_for_credentials() {
+    local namespace=$1
+    echo "Waiting for S3 credentials in $namespace..."
+
+    for i in {1..6}; do
+        if kubectl get secret -n $namespace mlpipeline-minio-artifact >/dev/null 2>&1; then
+            echo "Credentials found"
+            return 0
+        fi
+        sleep 10
+    done
+
+    echo "Error: No credentials found"
+    return 1
+}
+
+# Get credentials for namespace
+get_credentials() {
+    local namespace=$1
+    local access_key=$(kubectl get secret -n $namespace mlpipeline-minio-artifact -o jsonpath='{.data.accesskey}' | base64 -d)
+    local secret_key=$(kubectl get secret -n $namespace mlpipeline-minio-artifact -o jsonpath='{.data.secretkey}' | base64 -d)
+    echo "$access_key:$secret_key"
+}
+
+# Setup port forward to SeaweedFS
+setup_port_forward() {
+    if [ -n "$PORT_FORWARD_PID" ]; then
+        return 0  # Already running
+    fi
+
+    echo "Setting up port-forward..."
+    local pod=$(kubectl get pod -n kubeflow -l app=seaweedfs -o jsonpath='{.items[0].metadata.name}')
+    kubectl port-forward -n kubeflow pod/$pod 8333:8333 >/dev/null 2>&1 &
+    PORT_FORWARD_PID=$!
+    sleep 3
+}
+
+# Upload test file
+upload_file() {
+    local namespace=$1
+    echo "Uploading test file to $namespace..."
+
+    local credentials=$(get_credentials $namespace)
+    local access_key=$(echo $credentials | cut -d: -f1)
+    local secret_key=$(echo $credentials | cut -d: -f2)
+
+    setup_port_forward
+
+    python3 tests/gh-actions/s3_test_helper.py upload \
+        --access-key "$access_key" \
+        --secret-key "$secret_key" \
+        --endpoint-url "http://localhost:8333" \
+        --bucket "mlpipeline" \
+        --key "private-artifacts/$namespace/test-file.txt" \
+        --content "Test file for $namespace"
+}
+
+# Test unauthorized access
+test_unauthorized_access() {
+    local from_namespace=$1
+    local target_namespace=$2
+
+    echo "Testing unauthorized access from $from_namespace to $target_namespace..."
+
+    local credentials=$(get_credentials $from_namespace)
+    local access_key=$(echo $credentials | cut -d: -f1)
+    local secret_key=$(echo $credentials | cut -d: -f2)
+
+    setup_port_forward
+
+    # Try to access the other namespace's file
+    # Note: Python script returns 0 when access is denied (good), 1 when access succeeds (bad)
+    if python3 tests/gh-actions/s3_test_helper.py download \
+        --access-key "$access_key" \
+        --secret-key "$secret_key" \
+        --endpoint-url "http://localhost:8333" \
+        --bucket "mlpipeline" \
+        --key "private-artifacts/$target_namespace/test-file.txt"; then
+
+        echo "Security OK: Access denied as expected"
+        return 0
+    else
+        echo "SECURITY ISSUE: Unauthorized access successful!"
+        return 1
+    fi
+}
+
+# Main test function
+main() {
+    echo "Starting security test..."
+
+    # Create test profiles
+    create_profiles
+
+    # Wait for credentials to be created
+    echo "Waiting for profile controller to create credentials..."
+    sleep 30
+
+    wait_for_credentials "test-profile-1" || {
+        echo "Failed to get credentials for test-profile-1"
+        exit 1
+    }
+
+    wait_for_credentials "test-profile-2" || {
+        echo "Failed to get credentials for test-profile-2"
+        exit 1
+    }
+
+    # Upload file to first namespace
+    upload_file "test-profile-1" || {
+        echo "Failed to upload file"
+        exit 1
+    }
+
+    # Test unauthorized access
+    if test_unauthorized_access "test-profile-2" "test-profile-1"; then
+        echo "SECURITY TEST PASSED: No unauthorized access detected"
+    else
+        echo "SECURITY TEST FAILED: Unauthorized access detected"
+        echo "This indicates a security vulnerability in the SeaweedFS setup"
+        exit 1
+    fi
+}
+
+main