BioNeMo Sub-Package Workflow #141
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: BioNeMo Sub-Package Workflow | |
| on: | |
| # To test or publish sub-packages or adjustments to this workflow that are branched in PR's, manually dispatch this workflow on the PR's branch here: https://github.com/NVIDIA/bionemo-framework/actions/workflows/bionemo-subpackage-ci.yml. | |
| workflow_dispatch: | |
| inputs: | |
| subpackages: | |
| description: "BioNeMo sub-packages (comma-separated) to test or publish." | |
| required: true | |
| type: string | |
| test: | |
| description: "Test the sub-packages before publishing to PyPI. Strongly recommended for production releases to PyPI. Can be disabled when staging sub-packages on Test PyPI or publishing circular dependencies to PyPI." | |
| required: false | |
| type: boolean | |
| default: true | |
| publish: | |
| description: "Publish the built package to PyPI. If testing is specified, requires that all sub-package tests succeed based on dependencies published to Test PyPI or PyPI." | |
| required: false | |
| type: boolean | |
| default: false | |
| pypi: | |
| description: "Publish to PyPI instead of Test PyPI." | |
| required: false | |
| type: boolean | |
| default: false | |
| version_overwrite: | |
| description: "Overwrite the published version of the sub-package. (Sets skip-existing to False.)" | |
| required: false | |
| type: boolean | |
| default: false | |
| python_version: | |
| description: "Python version to use" | |
| required: false | |
| type: string | |
| default: "3.12" | |
| cuda_version: | |
| description: "CUDA version to use (format: X.Y.Z)" | |
| required: false | |
| type: string | |
| default: "12.8.1" | |
| ubuntu_version: | |
| description: "Ubuntu version to use" | |
| required: false | |
| type: string | |
| default: "22.04" | |
| jobs: | |
| configure-workflow-packages: | |
| name: "[Configure Workflow Packages] Identify sub-packages for testing and publishing." | |
| runs-on: ubuntu-latest | |
| outputs: | |
| workflow_packages: ${{ steps.parse-dispatch-packages.outputs.dispatch_packages }} | |
| steps: | |
| - id: parse-dispatch-packages | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| name: Parse the sub-packages specified in the workflow dispatch. | |
| run: | | |
| # Send the parsed list of sub-packages to the next job. | |
| dispatch_packages=$(echo '${{ github.event.inputs.subpackages }}' | jq -R -c 'split(",")') | |
| echo "dispatch_packages=$dispatch_packages" >> "$GITHUB_OUTPUT" | |
| echo "[BioNeMo Sub-Package CI] Sub-packages to stage: $dispatch_packages" | |
| install-and-test: | |
| needs: configure-workflow-packages | |
| # Check if the previous job has any staged packages to test and publish. | |
| if: ${{ needs.configure-workflow-packages.outputs.workflow_packages != '[]' }} | |
| strategy: | |
| matrix: | |
| package: ${{ fromJson(needs.configure-workflow-packages.outputs.workflow_packages) }} | |
| fail-fast: false # Prevent all matrix jobs from failing if one fails. | |
| name: "[${{ matrix.package }}] Install and test sub-package." | |
| runs-on: linux-amd64-gpu-l4-latest-1 | |
| container: # GPU jobs must run in a container. Use a fresh CUDA base container for package installation and testing. | |
| image: nvidia/cuda:${{ github.event.inputs.cuda_version }}-cudnn-devel-ubuntu${{ github.event.inputs.ubuntu_version }} | |
| steps: | |
| # Silently skip all steps if testing is disabled, which does not block building or publishing. | |
| - name: Install git and system dependencies. | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| apt-get install -y lsb-release # No longer pre-installed in Ubuntu>=22.04. | |
| apt-get install -y build-essential # For installing C build tools, like GCC and make. | |
| - uses: actions/checkout@v4 | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| with: | |
| fetch-depth: 0 | |
| submodules: "recursive" | |
| - uses: actions/setup-python@v5 | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| with: | |
| python-version: ${{ github.event.inputs.python_version }} | |
| - id: install-subpackage-core | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Install sub-package. | |
| run: | | |
| # Install sub-package and dependencies. | |
| export PIP_ROOT_USER_ACTION=ignore | |
| pip install --upgrade pip setuptools uv | |
| # Install required core & optional [test] dependencies. | |
| uv pip install --no-cache --system pytest sub-packages/${{ matrix.package }}[test] | |
| - id: install-subpackage-post | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Install sub-package dependencies that need to be installed after the core dependencies. | |
| run: | | |
| # DEV: Post-install dependencies are configured in [project.optional-dependencies]. | |
| # `uv pip install --extra <optional-dependency> -r <pyproject.toml>` tracks | |
| # post-dependencies in the pyproject.toml and avoids installing core dependencies | |
| # redundantly, which causes errors with incompatible --config-setting. | |
| # Check version of PyTorch. | |
| python -c "import torch; print(torch.__version__)" | |
| # TransformerEngine | |
| uv pip install --no-cache --no-build-isolation --system --extra te -r sub-packages/${{ matrix.package }}/pyproject.toml || echo "[BioNeMo Sub-Package CI] TE will not be installed." | |
| # Apex | |
| # NOTE: --cpp_ext and --cuda_ext are required for building fused Apex kernels. | |
| uv pip install --no-cache --no-build-isolation --system --config-setting="--build-option=--cpp_ext" --config-setting="--build-option=--cuda_ext" --extra apex -r sub-packages/${{ matrix.package }}/pyproject.toml || echo "[BioNeMo Sub-Package CI] Apex will not be installed." | |
| - id: test-dispatch-subpackage | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Test sub-package. | |
| run: pytest -vv sub-packages/${{ matrix.package }} | |
| build-pypi: | |
| # Build distributions from either the workflow dispatch or PR. | |
| # Validate building before merging or publishing. | |
| needs: [configure-workflow-packages, install-and-test] | |
| if: ${{ needs.configure-workflow-packages.outputs.workflow_packages != '[]' && github.event.inputs.publish == 'true' }} | |
| outputs: | |
| staged_packages: ${{ needs.configure-workflow-packages.outputs.workflow_packages }} | |
| strategy: | |
| matrix: | |
| package: ${{ fromJson(needs.configure-workflow-packages.outputs.workflow_packages) }} | |
| fail-fast: false # Prevent all matrix jobs from failing if one fails. | |
| name: "[${{ matrix.package }}] Build the sub-package." | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ github.event.inputs.python_version }} | |
| - id: build-package | |
| name: Build a binary wheel and a source tarball for the sub-package. | |
| run: | | |
| if [[ "${{ github.event.inputs.test }}" != "true" ]]; then | |
| # For untested sub-packages, append '-dev' to the version for PyPI. | |
| sed -i 's/[[:space:]]*$//' sub-packages/${{ matrix.package }}/VERSION | |
| sed -i 's/$/-dev/' sub-packages/${{ matrix.package }}/VERSION | |
| fi | |
| python -m pip install build | |
| python -m build sub-packages/${{ matrix.package }} | |
| - id: upload-distribution | |
| name: Upload distribution packages to the workflow. | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }}-dist | |
| path: sub-packages/${{ matrix.package }}/dist | |
| publish-to-pypi: | |
| needs: [build-pypi, install-and-test] | |
| # Require staged sub-package builds for publishing to PyPI. | |
| if: ${{ needs.build-pypi.result == 'success' }} | |
| strategy: | |
| matrix: | |
| package: ${{ fromJson(needs.build-pypi.outputs.staged_packages) }} | |
| fail-fast: false # Prevent all matrix jobs from failing if one fails. | |
| name: Publish ${{ matrix.package }} to PyPI. | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: ${{ github.event.inputs.pypi && 'pypi' || 'testpypi' }} | |
| url: ${{ github.event.inputs.pypi && format('https://pypi.org/p/{0}', matrix.package) || format('https://test.pypi.org/p/{0}', matrix.package) }} | |
| permissions: | |
| id-token: write | |
| steps: | |
| - id: download-distribution | |
| name: Download the built distribution. | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }}-dist | |
| path: sub-packages/${{ matrix.package }}/dist | |
| - id: publish-to-testpypi | |
| name: Publish distribution 📦 to Test PyPI for PR. | |
| if: ${{ github.event.inputs.pypi == 'false' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| verbose: true | |
| packages-dir: sub-packages/${{ matrix.package }}/dist | |
| repository-url: https://test.pypi.org/legacy/ | |
| skip-existing: ${{ github.event.inputs.version_overwrite }} | |
| - id: publish-to-pypi | |
| name: Publish distribution 📦 to PyPI for Workflow Dispatch. | |
| # To require testing before publishing to PyPI, add: ... && needs.install-and-test.result == 'success' | |
| # If testing is run but fails, the workflow will fail and not publish to PyPI (or Test PyPI). | |
| # We strongly recommend testing when publishing to production PyPI. | |
| if: ${{ github.event.inputs.pypi == 'true' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| verbose: true | |
| packages-dir: sub-packages/${{ matrix.package }}/dist | |
| skip-existing: ${{ github.event.inputs.version_overwrite }} |