E2E Tests v2 (t1) - Post-merge 684a1ceb7ce916e0c3b90e89191a1f0e6f2eef44 #841
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests - Post Merge (v2) | |
| run-name: "E2E Tests v2 (${{ github.event.inputs.tier || 't1' }}) - ${{ github.event_name == 'push' && format('Post-merge {0}', github.sha) || format('Manual run by @{0}', github.actor) }}" | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| # Skip workflow if only documentation or unrelated files changed | |
| paths-ignore: | |
| - '**.md' | |
| - 'apps/opik-documentation/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/e2e_tests_post_merge_v2.yml' | |
| workflow_dispatch: | |
| inputs: | |
| tier: | |
| type: choice | |
| description: 'Which tag tier to run (t1=smoke, t2=cuj, t3=nightly)' | |
| required: true | |
| default: 't1' | |
| options: | |
| - t1 | |
| - t2 | |
| - t3 | |
| notify_on_success: | |
| type: boolean | |
| description: 'Send Slack notification even on success' | |
| required: false | |
| default: false | |
| ALLURE_JOB_RUN_ID: | |
| description: 'ALLURE_JOB_RUN_ID service parameter. Leave blank.' | |
| required: false | |
| env: | |
| ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} | |
| ALLURE_ENDPOINT: https://comet.testops.cloud/ | |
| ALLURE_PROJECT_ID: 1 | |
| ALLURE_RESULTS: allure-results | |
| ALLURE_JOB_RUN_ID: ${{ github.event.inputs.ALLURE_JOB_RUN_ID }} | |
| OPIK_SENTRY_ENABLE: false | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| e2e-tests: | |
| name: "🧪 E2E v2 Tests (${{ github.event.inputs.tier || 't1' }})" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| outputs: | |
| test_result: ${{ steps.run-tests.outcome }} | |
| testops_url: ${{ steps.set-outputs.outputs.testops_url }} | |
| total_tests: ${{ steps.parse-results.outputs.total }} | |
| passed_tests: ${{ steps.parse-results.outputs.passed }} | |
| failed_tests: ${{ steps.parse-results.outputs.failed }} | |
| skipped_tests: ${{ steps.parse-results.outputs.skipped }} | |
| pr_author: ${{ steps.get-pr-info.outputs.author }} | |
| steps: | |
| # ======================================== | |
| # PHASE 1: Checkout & Environment Setup | |
| # ======================================== | |
| - name: "📥 Checkout repository" | |
| uses: actions/checkout@v7 | |
| - name: "🔍 Get PR/commit information" | |
| id: get-pr-info | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| COMMIT_AUTHOR: ${{ github.event.head_commit.author.username }} | |
| ACTOR: ${{ github.actor }} | |
| run: | | |
| if [ "${EVENT_NAME}" == "push" ]; then | |
| # For push events, get the author from the commit | |
| AUTHOR="${COMMIT_AUTHOR}" | |
| if [ -z "$AUTHOR" ]; then | |
| AUTHOR="${ACTOR}" | |
| fi | |
| else | |
| AUTHOR="${ACTOR}" | |
| fi | |
| echo "author=$AUTHOR" >> "$GITHUB_OUTPUT" | |
| echo "📝 Author: $AUTHOR" | |
| - name: "📦 Setup Node.js" | |
| uses: actions/setup-node@v7 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: tests_end_to_end/e2e/package-lock.json | |
| - name: "🐍 Setup uv & Python" | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: "📚 Install E2E test dependencies" | |
| working-directory: tests_end_to_end/e2e | |
| run: | | |
| npm ci | |
| npx playwright install --with-deps chromium | |
| - name: "📚 Sync bridge dependencies" | |
| working-directory: tests_end_to_end/e2e/services/opik-sdk-driver | |
| run: uv sync | |
| - name: "📊 Install allurectl" | |
| uses: allure-framework/setup-allurectl@v1 | |
| # ======================================== | |
| # PHASE 2: Build & Start Opik | |
| # ======================================== | |
| - name: "🔨 Build and start Opik server" | |
| env: | |
| OPIK_USAGE_REPORT_ENABLED: false | |
| COMPOSE_BAKE: false | |
| REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| echo "::group::Building Opik from branch: ${REF_NAME}" | |
| cd ${{ github.workspace }} | |
| TOGGLE_WELCOME_WIZARD_ENABLED="false" ./opik.sh --build | |
| echo "::endgroup::" | |
| echo "✅ Opik build complete" | |
| - name: "🔍 Verify Docker pods are running" | |
| run: | | |
| echo "::group::Checking Docker pods" | |
| chmod +x ./tests_end_to_end/installer_utils/check_docker_compose_pods.sh | |
| ./tests_end_to_end/installer_utils/check_docker_compose_pods.sh | |
| echo "::endgroup::" | |
| echo "✅ All Docker pods are running" | |
| - name: "🏥 Verify backend health" | |
| run: | | |
| echo "::group::Checking backend health" | |
| chmod +x ./tests_end_to_end/installer_utils/check_backend.sh | |
| ./tests_end_to_end/installer_utils/check_backend.sh | |
| echo "::endgroup::" | |
| echo "✅ Backend is healthy" | |
| # ======================================== | |
| # PHASE 3: Run E2E Tests | |
| # ======================================== | |
| - name: "🧪 Run E2E test suite" | |
| id: run-tests | |
| working-directory: tests_end_to_end/e2e | |
| env: | |
| OPIK_DEPLOYMENT: oss | |
| OPIK_BASE_URL: http://localhost:5173 | |
| OPIK_WORKSPACE: default | |
| WORKERS: 2 | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ALLURE_LAUNCH_NAME: "Opik v2 Post-Merge ${{ github.event.inputs.tier || 't1' }} - ${{ github.run_id }}" | |
| TIER: ${{ github.event.inputs.tier || 't1' }} | |
| run: | | |
| echo "========================================" | |
| echo "🧪 Running v2 tier: $TIER" | |
| echo "========================================" | |
| echo "" | |
| # Point both the allure-playwright reporter and allurectl at the same | |
| # absolute results dir, then stream results to TestOps via allurectl watch. | |
| export ALLURE_RESULTS="${{ github.workspace }}/tests_end_to_end/e2e/${{ env.ALLURE_RESULTS }}" | |
| # Create .allure directory to prevent testplan.json error | |
| mkdir -p .allure | |
| # Run tests with allurectl watch for TestOps integration | |
| # Capture output to extract the direct report URL | |
| set +e | |
| allurectl watch -- npm run "test:${TIER}" 2>&1 | tee allure_output.log | |
| TEST_EXIT_CODE=${PIPESTATUS[0]} | |
| set -e | |
| # Extract the direct job run URL from allurectl output | |
| if grep -q "Report link:" allure_output.log; then | |
| REPORT_URL=$(grep "Report link:" allure_output.log | head -1 | sed 's/.*Report link:[[:space:]]*//' | awk '{print $1}') | |
| echo "allure_report_url=${REPORT_URL}" >> "$GITHUB_OUTPUT" | |
| echo "📊 Captured Allure TestOps report URL: ${REPORT_URL}" | |
| else | |
| echo "allure_report_url=" >> "$GITHUB_OUTPUT" | |
| echo "⚠️ Could not capture direct report URL from allurectl output" | |
| fi | |
| echo "" | |
| echo "========================================" | |
| echo "Test run completed with exit code: $TEST_EXIT_CODE" | |
| echo "========================================" | |
| exit "$TEST_EXIT_CODE" | |
| # ======================================== | |
| # PHASE 4: Process Results | |
| # ======================================== | |
| - name: "📊 Parse test results" | |
| id: parse-results | |
| if: always() | |
| working-directory: tests_end_to_end/e2e | |
| run: | | |
| echo "::group::Parsing test results" | |
| if [ -f "test-results/results.json" ]; then | |
| echo "Found results.json, parsing..." | |
| TOTAL=$(jq '.stats.expected + .stats.unexpected + .stats.flaky + .stats.skipped' test-results/results.json 2>/dev/null || echo "0") | |
| PASSED=$(jq '.stats.expected' test-results/results.json 2>/dev/null || echo "0") | |
| FAILED=$(jq '.stats.unexpected' test-results/results.json 2>/dev/null || echo "0") | |
| SKIPPED=$(jq '.stats.skipped' test-results/results.json 2>/dev/null || echo "0") | |
| FLAKY=$(jq '.stats.flaky' test-results/results.json 2>/dev/null || echo "0") | |
| echo "Results: Total=$TOTAL, Passed=$PASSED, Failed=$FAILED, Skipped=$SKIPPED, Flaky=$FLAKY" | |
| else | |
| echo "⚠️ results.json not found, using defaults" | |
| TOTAL="0" | |
| PASSED="0" | |
| FAILED="0" | |
| SKIPPED="0" | |
| FLAKY="0" | |
| fi | |
| { | |
| echo "total=$TOTAL" | |
| echo "passed=$PASSED" | |
| echo "failed=$FAILED" | |
| echo "skipped=$SKIPPED" | |
| echo "flaky=$FLAKY" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "::endgroup::" | |
| - name: "🔗 Set output URLs" | |
| id: set-outputs | |
| if: always() | |
| run: | | |
| # Use direct report URL if captured, otherwise fall back to launches page | |
| DIRECT_URL="${{ steps.run-tests.outputs.allure_report_url }}" | |
| if [ -n "$DIRECT_URL" ]; then | |
| TESTOPS_URL="$DIRECT_URL" | |
| echo "📊 Using direct TestOps report URL" | |
| else | |
| TESTOPS_URL="${{ env.ALLURE_ENDPOINT }}project/${{ env.ALLURE_PROJECT_ID }}/launches" | |
| echo "📊 Using fallback launches page URL" | |
| fi | |
| echo "testops_url=$TESTOPS_URL" >> "$GITHUB_OUTPUT" | |
| # ======================================== | |
| # PHASE 5: Generate Summary | |
| # ======================================== | |
| - name: "📋 Generate job summary" | |
| if: always() | |
| env: | |
| TIER: ${{ github.event.inputs.tier || 't1' }} | |
| TESTOPS_URL: ${{ steps.set-outputs.outputs.testops_url }} | |
| TOTAL: ${{ steps.parse-results.outputs.total }} | |
| PASSED: ${{ steps.parse-results.outputs.passed }} | |
| FAILED: ${{ steps.parse-results.outputs.failed }} | |
| SKIPPED: ${{ steps.parse-results.outputs.skipped }} | |
| RUN_TESTS_OUTCOME: ${{ steps.run-tests.outcome }} | |
| REF_NAME: ${{ github.ref_name }} | |
| REPOSITORY: ${{ github.repository }} | |
| COMMIT_SHA: ${{ github.sha }} | |
| TRIGGER_LABEL: ${{ github.event_name == 'push' && 'Post-merge (push to main)' || 'Manual dispatch' }} | |
| AUTHOR: ${{ steps.get-pr-info.outputs.author }} | |
| run: | | |
| # Redirect stdout to the summary file for the remainder of this step | |
| exec >> "$GITHUB_STEP_SUMMARY" | |
| # Header | |
| echo "# 🧪 E2E v2 Test Results" | |
| echo "" | |
| # Status banner | |
| if [ "${RUN_TESTS_OUTCOME}" == "success" ]; then | |
| echo "> ✅ **All tests passed!**" | |
| elif [ "${RUN_TESTS_OUTCOME}" == "failure" ]; then | |
| echo "> ❌ **$FAILED test(s) failed** - Please review the results below" | |
| else | |
| echo "> ⚠️ **Test run status: ${RUN_TESTS_OUTCOME}**" | |
| fi | |
| echo "" | |
| # Configuration table | |
| echo "## 📝 Configuration" | |
| echo "" | |
| echo "| Property | Value |" | |
| echo "|----------|-------|" | |
| echo "| **Tier** | \`$TIER\` |" | |
| echo "| **Branch** | \`${REF_NAME}\` |" | |
| echo "| **Commit** | [\`${COMMIT_SHA:0:7}\`](https://github.com/${REPOSITORY}/commit/${COMMIT_SHA}) |" | |
| echo "| **Trigger** | ${TRIGGER_LABEL} |" | |
| echo "| **Author** | @${AUTHOR} |" | |
| echo "" | |
| # Results table | |
| echo "## 📊 Results" | |
| echo "" | |
| echo "| Status | Count |" | |
| echo "|--------|------:|" | |
| echo "| ✅ Passed | $PASSED |" | |
| echo "| ❌ Failed | $FAILED |" | |
| echo "| ⏭️ Skipped | $SKIPPED |" | |
| echo "| **Total** | **$TOTAL** |" | |
| echo "" | |
| # Links section | |
| echo "## 🔗 Quick Links" | |
| echo "" | |
| echo "| Resource | Link |" | |
| echo "|----------|------|" | |
| echo "| 📊 **TestOps Dashboard** | [View detailed results, traces & history]($TESTOPS_URL) |" | |
| echo "| 📄 **Playwright Report** | [Download from Artifacts below](#artifacts) |" | |
| echo "" | |
| # Debugging section (only on failure) | |
| if [ "${RUN_TESTS_OUTCOME}" == "failure" ]; then | |
| echo "## 🔧 Debugging Guide" | |
| echo "" | |
| echo "### Quick steps to investigate failures:" | |
| echo "" | |
| echo "1. **Open TestOps** → Click the link above to see detailed failure traces, screenshots, and video recordings" | |
| echo "2. **Download Playwright Report** → The HTML report includes step-by-step traces for each test" | |
| echo "3. **Check test logs** → Expand the 'Run E2E test suite' step above for console output" | |
| echo "" | |
| echo "### Common failure causes:" | |
| echo "- UI element selectors changed (missing data-testid)" | |
| echo "- API response format changed" | |
| echo "- Timing/race conditions" | |
| echo "- New feature requiring test updates" | |
| fi | |
| # ======================================== | |
| # PHASE 6: Upload Artifacts | |
| # ======================================== | |
| - name: "📤 Upload Allure results" | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: allure-results-v2 | |
| path: tests_end_to_end/e2e/allure-results | |
| retention-days: 14 | |
| - name: "📤 Upload Playwright report" | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: playwright-report-v2 | |
| path: tests_end_to_end/e2e/playwright-report | |
| retention-days: 14 | |
| - name: "📤 Upload test results JSON" | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-json-v2 | |
| path: tests_end_to_end/e2e/test-results/results.json | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| - name: "🧹 Stop Opik server" | |
| if: always() | |
| run: | | |
| echo "Stopping Opik server..." | |
| ./opik.sh --stop || true | |
| echo "✅ Cleanup complete" | |
| notify-slack: | |
| name: "📢 Slack Notification" | |
| runs-on: ubuntu-latest | |
| needs: e2e-tests | |
| if: | | |
| always() && | |
| needs.e2e-tests.result != 'cancelled' && | |
| (needs.e2e-tests.result == 'failure' || github.event.inputs.notify_on_success == 'true') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| sparse-checkout: .github/scripts | |
| sparse-checkout-cone-mode: false | |
| - name: Resolve Slack mentions | |
| id: resolve-slack-mentions | |
| env: | |
| SLACK_USER_MAPPING: ${{ secrets.SLACK_USER_MAPPING }} | |
| SLACK_QA_LEAD_ID: ${{ secrets.SLACK_QA_LEAD_ID }} | |
| run: | | |
| AUTHOR="${{ needs.e2e-tests.outputs.pr_author }}" | |
| IS_FAILURE="${{ needs.e2e-tests.result == 'failure' }}" | |
| if [ "$IS_FAILURE" == "true" ]; then | |
| if [ -n "$SLACK_QA_LEAD_ID" ]; then | |
| QA_MENTION="<@$SLACK_QA_LEAD_ID>" | |
| else | |
| QA_MENTION="" | |
| fi | |
| AUTHOR_MENTION="" | |
| if [ -n "$SLACK_USER_MAPPING" ]; then | |
| AUTHOR_SLACK_ID=$(echo "$SLACK_USER_MAPPING" | jq -r --arg user "$AUTHOR" '.[$user] // empty' 2>/dev/null || echo "") | |
| if [ -n "$AUTHOR_SLACK_ID" ] && [ "$AUTHOR_SLACK_ID" != "$SLACK_QA_LEAD_ID" ]; then | |
| AUTHOR_MENTION="<@$AUTHOR_SLACK_ID>" | |
| fi | |
| fi | |
| MENTIONS="${QA_MENTION:+$QA_MENTION }${AUTHOR_MENTION}" | |
| echo "mentions=${MENTIONS% }" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "mentions=" >> "$GITHUB_OUTPUT" | |
| fi | |
| echo "author_display=$AUTHOR" >> "$GITHUB_OUTPUT" | |
| - name: Send Slack notification | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| TEST_RESULT: ${{ needs.e2e-tests.result }} | |
| SUITE_NAME: "v2 ${{ github.event.inputs.tier || 't1' }}" | |
| PASSED_TESTS: ${{ needs.e2e-tests.outputs.passed_tests }} | |
| FAILED_TESTS: ${{ needs.e2e-tests.outputs.failed_tests }} | |
| TESTOPS_URL: ${{ needs.e2e-tests.outputs.testops_url }} | |
| AUTHOR_DISPLAY: ${{ steps.resolve-slack-mentions.outputs.author_display }} | |
| MENTIONS: ${{ steps.resolve-slack-mentions.outputs.mentions }} | |
| run: .github/scripts/notify-slack-e2e-post-merge.sh |