fix(ingest/transformer): MultipleAspectTransformer silently drops non-matching MCPWs #57869
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build & test | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - releases/** | |
| - hotfixes/** | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| - ".github/**" | |
| - "!.github/workflows/build-and-test.yml" | |
| - "!.github/actions/ensure-codegen-updated" | |
| - "!.github/scripts/send_failed_tests_to_posthog.py" | |
| pull_request: | |
| branches: | |
| - "**" | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| - ".github/**" | |
| - "!.github/workflows/build-and-test.yml" | |
| - "!.github/actions/ensure-codegen-updated" | |
| - "!.github/scripts/send_failed_tests_to_posthog.py" | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 0 * * *" # Run at midnight UTC every day | |
| release: | |
| types: [published] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DEPOT_PROJECT_ID: "${{ vars.DEPOT_PROJECT_ID }}" | |
| HAS_DEPOT_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'depot') }} | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' || github.event_name != 'pull_request' }} | |
| ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }} | |
| backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' || github.event_name != 'pull_request'}} | |
| docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' || github.event_name != 'pull_request' }} | |
| frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }} | |
| ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }} | |
| kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }} | |
| runner_type: ${{ steps.set-runner.outputs.runner_type }} | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| - uses: ./.github/actions/ci-optimization | |
| id: ci-optimize | |
| - name: Determine runner type | |
| id: set-runner | |
| # Use depot runners only when a maintainer has explicitly added the "depot" label. | |
| # vars.DEPOT_PROJECT_ID is not accessible to fork PRs, so we rely solely on the label. | |
| run: | | |
| if [[ "${{ env.HAS_DEPOT_LABEL }}" == "true" ]]; then | |
| echo "runner_type=depot-ubuntu-latest-4" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "runner_type=ubuntu-latest" >> "$GITHUB_OUTPUT" | |
| fi | |
| build-and-test: | |
| runs-on: ${{ needs.setup.outputs.runner_type }} | |
| timeout-minutes: 60 | |
| needs: setup | |
| if: ${{ needs.setup.outputs.backend_change == 'true' }} | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| - name: Free up disk space | |
| uses: ./.github/actions/free-disk-space | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: "3.10" | |
| - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} | |
| - name: Install dependencies | |
| run: ./metadata-ingestion/scripts/install_deps.sh | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: "zulu" | |
| java-version: 17 | |
| - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4 | |
| - name: Disk Space Analysis | |
| run: | | |
| echo "=== Disk Usage Overview ===" | |
| df -h | |
| echo -e "\n=== Docker Disk Usage ===" | |
| docker system df -v | |
| - name: Gradle build (and test) for NOT metadata ingestion | |
| # datahub-schematron:cli excluded due to dependency on metadata-ingestion | |
| run: | | |
| ./gradlew build \ | |
| -x :metadata-ingestion:build \ | |
| -x :metadata-ingestion:check \ | |
| -x :docs-website:build \ | |
| -x :metadata-integration:java:spark-lineage:test \ | |
| -x :metadata-io:test \ | |
| -x :metadata-ingestion-modules:airflow-plugin:build \ | |
| -x :metadata-ingestion-modules:airflow-plugin:check \ | |
| -x :metadata-ingestion-modules:dagster-plugin:build \ | |
| -x :metadata-ingestion-modules:dagster-plugin:check \ | |
| -x :metadata-ingestion-modules:prefect-plugin:build \ | |
| -x :metadata-ingestion-modules:prefect-plugin:check \ | |
| -x :metadata-ingestion-modules:gx-plugin:build \ | |
| -x :metadata-ingestion-modules:gx-plugin:check \ | |
| -x :datahub-frontend:build \ | |
| -x :datahub-web-react:build \ | |
| -x :metadata-integration:java:datahub-schematron:cli:test | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Disk Space Analysis | |
| run: | | |
| echo "=== Disk Usage Overview ===" | |
| df -h | |
| echo -e "\n=== Docker Disk Usage ===" | |
| docker system df -v | |
| - name: Gradle compile (jdk8) for legacy Spark | |
| run: | | |
| ./gradlew -PjavaClassVersionDefault=17 :metadata-integration:java:spark-lineage:compileJava | |
| - name: Gather coverage files | |
| run: | | |
| { | |
| echo "BACKEND_FILES=$(find ./build/coverage-reports/ -type f | grep -E '(metadata-models|entity-registry|datahub-graphql-core|metadata-io|metadata-jobs|metadata-utils|metadata-service|medata-dao-impl|metadata-operation|li-utils|metadata-integration|metadata-events|metadata-auth|ingestion-scheduler|notifications|datahub-upgrade)' | xargs | tr ' ' ',')" | |
| } >> "$GITHUB_ENV" | |
| - name: Report test results | |
| if: (!cancelled()) | |
| uses: ./.github/actions/report-test-results | |
| with: | |
| artifact-name: build-and-test-except_metadata_ingestion-UTC | |
| test-results-paths: | | |
| **/build/reports/tests/test/** | |
| **/build/test-results/test/** | |
| **/junit.*.xml | |
| !**/binary/** | |
| junit-file-globs: | | |
| **/build/test-results/test/**/*.xml | |
| **/junit.*.xml | |
| - name: Send failed test metrics to PostHog | |
| if: failure() | |
| continue-on-error: true | |
| env: | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} | |
| GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} | |
| run: | | |
| if [ -z "$POSTHOG_API_KEY" ]; then | |
| echo "⚠️ POSTHOG_API_KEY not configured, skipping test failure metrics" | |
| exit 0 | |
| fi | |
| TEMP_DIR=$(mktemp -d) | |
| mkdir -p "$TEMP_DIR/test-results" | |
| find . -name "*.xml" -path "*/build/test-results/*" -exec cp {} "$TEMP_DIR/test-results/" \; 2>/dev/null || true | |
| find . -name "junit.*.xml" -exec cp {} "$TEMP_DIR/test-results/" \; 2>/dev/null || true | |
| python3 .github/scripts/send_failed_tests_to_posthog.py \ | |
| --input-dir "$TEMP_DIR/test-results" \ | |
| --posthog-api-key "$POSTHOG_API_KEY" \ | |
| --posthog-host "${POSTHOG_HOST:-https://app.posthog.com}" \ | |
| --repository "${{ github.repository }}" \ | |
| --workflow-name "${{ github.workflow }}" \ | |
| --branch "${GH_HEAD_REF}" \ | |
| --run-id "${{ github.run_id }}" \ | |
| --run-attempt "${{ github.run_attempt }}" \ | |
| --command "except_metadata_ingestion" \ | |
| --timezone "UTC" | |
| rm -rf "$TEMP_DIR" | |
| - name: Ensure codegen is updated | |
| uses: ./.github/actions/ensure-codegen-updated | |
| - name: Upload backend coverage to Codecov | |
| if: ${{ github.event_name != 'release' }} | |
| uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ${{ env.BACKEND_FILES }} | |
| disable_search: true | |
| #handle_no_reports_found: true | |
| fail_ci_if_error: false | |
| flags: backend | |
| name: except_metadata_ingestion | |
| verbose: true | |
| - name: Upload backend coverage to Codecov on release | |
| if: ${{ github.event_name == 'release' }} | |
| uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ${{ env.BACKEND_FILES }} | |
| disable_search: true | |
| #handle_no_reports_found: true | |
| fail_ci_if_error: false | |
| flags: backend | |
| name: except_metadata_ingestion | |
| verbose: true | |
| override_branch: ${{ github.head_ref || github.ref_name }} | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() && github.event_name != 'release' }} | |
| uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload test results to Codecov on release | |
| if: ${{ !cancelled() && github.event_name == 'release' }} | |
| uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| override_branch: ${{ github.head_ref || github.ref_name }} | |
| frontend-build: | |
| runs-on: ${{ needs.setup.outputs.runner_type }} | |
| timeout-minutes: 30 | |
| needs: setup | |
| if: ${{ needs.setup.outputs.frontend_change == 'true' }} | |
| steps: | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| - name: Free up disk space | |
| uses: ./.github/actions/free-disk-space | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: "3.10" | |
| - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} | |
| - name: Install dependencies | |
| run: ./metadata-ingestion/scripts/install_deps.sh | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: "zulu" | |
| java-version: 17 | |
| - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4 | |
| - name: Gradle build and test for frontend (no JS tests) | |
| run: | | |
| ./gradlew :datahub-frontend:build :datahub-web-react:build \ | |
| -x :datahub-web-react:yarnTest | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Ensure codegen is updated | |
| uses: ./.github/actions/ensure-codegen-updated | |
| - name: Upload frontend build artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: frontend-build-artifacts | |
| retention-days: 1 | |
| path: | | |
| datahub-web-react/src/**/*.generated.ts | |
| datahub-frontend/build | |
| frontend-test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| timezone: ["UTC", "America/New_York"] | |
| frontend_shard: [1, 2, 3] | |
| runs-on: ${{ needs.setup.outputs.runner_type }} | |
| timeout-minutes: 30 | |
| needs: [setup, frontend-build] | |
| steps: | |
| - uses: szenius/set-timezone@1f9716b0f7120e344f0c62bb7b1ee98819aefd42 # v2.0 | |
| with: | |
| timezoneLinux: ${{ matrix.timezone }} | |
| - name: Check out the repo | |
| uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7 | |
| - name: Free up disk space | |
| uses: ./.github/actions/free-disk-space | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: "3.10" | |
| - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} | |
| - name: Install dependencies | |
| run: ./metadata-ingestion/scripts/install_deps.sh | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: "zulu" | |
| java-version: 17 | |
| - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4 | |
| - name: Download frontend build artifacts | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 | |
| with: | |
| name: frontend-build-artifacts | |
| - name: Gradle test for frontend (shard ${{ matrix.frontend_shard }}/3) | |
| run: | | |
| ./gradlew :datahub-web-react:yarnTest \ | |
| -PvitestShard="${{ matrix.frontend_shard }}/3" | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Gather coverage files | |
| run: | | |
| { | |
| echo "FRONTEND_FILES=$(find ./build/coverage-reports/ -type f | grep -E '(datahub-frontend|datahub-web-react).*\.(xml|json)$' | xargs | tr ' ' ',')" | |
| } >> "$GITHUB_ENV" | |
| - name: Generate tz artifact name | |
| run: echo "NAME_TZ=$(echo "${{ matrix.timezone }}" | tr '/' '-')" >> "$GITHUB_ENV" | |
| - name: Report test results | |
| if: (!cancelled()) | |
| uses: ./.github/actions/report-test-results | |
| with: | |
| artifact-name: build-and-test-frontend-${{ env.NAME_TZ }}-shard${{ matrix.frontend_shard }} | |
| test-results-paths: | | |
| **/build/reports/tests/test/** | |
| **/build/test-results/test/** | |
| **/junit.*.xml | |
| !**/binary/** | |
| junit-file-globs: | | |
| **/build/test-results/test/**/*.xml | |
| **/junit.*.xml | |
| - name: Send failed test metrics to PostHog | |
| if: failure() | |
| continue-on-error: true | |
| env: | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} | |
| GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} | |
| run: | | |
| if [ -z "$POSTHOG_API_KEY" ]; then | |
| echo "⚠️ POSTHOG_API_KEY not configured, skipping test failure metrics" | |
| exit 0 | |
| fi | |
| TEMP_DIR=$(mktemp -d) | |
| mkdir -p "$TEMP_DIR/test-results" | |
| find . -name "*.xml" -path "*/build/test-results/*" -exec cp {} "$TEMP_DIR/test-results/" \; 2>/dev/null || true | |
| find . -name "junit.*.xml" -exec cp {} "$TEMP_DIR/test-results/" \; 2>/dev/null || true | |
| python3 .github/scripts/send_failed_tests_to_posthog.py \ | |
| --input-dir "$TEMP_DIR/test-results" \ | |
| --posthog-api-key "$POSTHOG_API_KEY" \ | |
| --posthog-host "${POSTHOG_HOST:-https://app.posthog.com}" \ | |
| --repository "${{ github.repository }}" \ | |
| --workflow-name "${{ github.workflow }}" \ | |
| --branch "${GH_HEAD_REF}" \ | |
| --run-id "${{ github.run_id }}" \ | |
| --run-attempt "${{ github.run_attempt }}" \ | |
| --command "frontend" \ | |
| --timezone "${{ matrix.timezone }}" | |
| rm -rf "$TEMP_DIR" | |
| - name: Upload frontend coverage to Codecov | |
| if: ${{ github.event_name != 'release' }} | |
| uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ${{ env.FRONTEND_FILES }} | |
| disable_search: true | |
| #handle_no_reports_found: true | |
| fail_ci_if_error: false | |
| flags: frontend | |
| name: frontend-shard${{ matrix.frontend_shard }} | |
| verbose: true | |
| - name: Upload frontend coverage to Codecov on Release | |
| if: ${{ github.event_name == 'release' }} | |
| uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ${{ env.FRONTEND_FILES }} | |
| disable_search: true | |
| #handle_no_reports_found: true | |
| fail_ci_if_error: false | |
| flags: frontend | |
| name: frontend-shard${{ matrix.frontend_shard }} | |
| verbose: true | |
| override_branch: ${{ github.head_ref || github.ref_name }} | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() && github.event_name != 'release' }} | |
| uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload test results to Codecov on release | |
| if: ${{ !cancelled() && github.event_name == 'release' }} | |
| uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| override_branch: ${{ github.head_ref || github.ref_name }} | |
| event-file: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Upload | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: Event File | |
| path: ${{ github.event_path }} |