Add Slack Notifications #5020
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| - "**" | |
| pull_request: {} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| webknossos: ${{ steps.filter.outputs.webknossos || github.ref == 'refs/heads/master' }} | |
| cluster_tools: ${{ steps.filter.outputs.cluster_tools || github.ref == 'refs/heads/master' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| webknossos: | |
| - 'webknossos/**' | |
| cluster_tools: | |
| - 'cluster_tools/**' | |
| cluster_tools_slurm: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "cluster_tools/uv.lock" | |
| - run: uv python install ${{ matrix.python-version }} | |
| - name: Start Docker Cluster | |
| run: cd ./dockered-slurm && docker compose up -d | |
| - name: Log Core Container | |
| run: | | |
| for name in "slurmctld" "c1" "c2"; do | |
| docker logs "$name" | |
| done | |
| - name: Install UV dependencies | |
| run: | | |
| for name in "slurmctld" "c1" "c2"; do | |
| docker exec -w /cluster_tools "$name" bash -c "uv sync --frozen" | |
| done | |
| - name: "Run Tests (test_all, test_slurm) without modified slurm.conf" | |
| run: | | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| -e PYTEST_EXECUTORS=slurm \ | |
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_all.py test_slurm.py -m 'not requires_modified_slurm_config'" | |
| - name: "Run Tests (test_deref_main)" | |
| run: | | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| slurmctld bash -c "uv run --frozen python test_deref_main.py" | |
| - name: Update Slurm Config | |
| run: | | |
| echo "MaxArraySize=2" >> ./dockered-slurm/slurm.conf | |
| sed "s/JobAcctGatherFrequency=30/JobAcctGatherFrequency=1/g" ./dockered-slurm/slurm.conf > ./dockered-slurm/slurm.conf.tmp | |
| mv ./dockered-slurm/slurm.conf.tmp ./dockered-slurm/slurm.conf | |
| - name: Restart Slurm Cluster | |
| run: cd ./dockered-slurm && docker compose restart slurmctld c1 c2 | |
| - name: "Run Tests (test_all, test_slurm) with modified slurn.conf" | |
| run: | | |
| # Run tests requiring a modified slurm config | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| -e PYTEST_EXECUTORS=slurm \ | |
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_slurm.py -m 'requires_modified_slurm_config'" | |
| cluster_tools_multiprocessing: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "cluster_tools/uv.lock" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install dependencies (without docker) | |
| run: uv sync --frozen | |
| - name: Check typing | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: ./typecheck.sh | |
| - name: Check formatting | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: ./format.sh check | |
| - name: Lint code | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: ./lint.sh | |
| - name: Run multiprocessing tests | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=multiprocessing,sequential,multiprocessing_with_pickling,sequential_with_pickling \ | |
| uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py | |
| cluster_tools_kubernetes: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "cluster_tools/uv.lock" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Setup Kubernetes-in-Docker | |
| run: | | |
| curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 | |
| chmod +x ./kind | |
| sed -i "s#__PATH__#$(pwd)#g" tests/cluster-config.yaml | |
| ./kind create cluster --config=tests/cluster-config.yaml | |
| ./kind export kubeconfig | |
| docker build \ | |
| --build-arg PYTHON_VERSION=${{ matrix.python-version }} \ | |
| -f tests/Dockerfile \ | |
| -t scalableminds/cluster-tools:latest \ | |
| . | |
| ./kind load docker-image scalableminds/cluster-tools:latest | |
| - name: Install dependencies (without docker) | |
| run: uv sync --all-extras --frozen | |
| - name: "Run Kubernetes" | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=kubernetes uv run --frozen python -m pytest -sv test_all.py test_kubernetes.py | |
| cluster_tools_dask: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "cluster_tools/uv.lock" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install dependencies (without docker) | |
| run: uv sync --all-extras --frozen | |
| - name: "Run Dask" | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=dask uv run --frozen python -m pytest -sv test_all.py test_dask.py | |
| webknossos_linux: | |
| needs: changes | |
| if: | | |
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | |
| ${{ needs.changes.outputs.webknossos == 'true' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| group: [1, 2, 3] | |
| fail-fast: false | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "webknossos/uv.lock" | |
| - name: Install proxay | |
| run: npm install -g proxay | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Check formatting | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./format.sh check | |
| - name: Lint code | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./lint.sh | |
| - name: Check typing | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./typecheck.sh | |
| - name: Patch Python standard library to assert that fork is not allowed | |
| run: | | |
| sed -i '/def _launch/a\ \ \ \ \ \ \ \ raise Exception("fork is not allowed.")' /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py | |
| cat /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py | |
| - name: Python tests (with coverage) | |
| if: ${{ matrix.python-version == '3.11' }} | |
| timeout-minutes: 30 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| COVERAGE_FILE: "~/.local/share/coverage/${{ matrix.group }}.coverage" | |
| run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration --cov=webknossos --cov-report= | |
| - name: Python tests | |
| if: ${{ matrix.python-version != '3.11' }} | |
| timeout-minutes: 30 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | |
| - name: Persist coverage reports | |
| if: ${{ matrix.python-version == '3.11' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-${{ matrix.group }} | |
| path: "~/.local/share/coverage/${{ matrix.group }}.coverage" | |
| retention-days: 1 # Automatically deletes after 1 day | |
| - name: Check if git is dirty | |
| run: | | |
| git diff --no-ext-diff --exit-code | |
| [[ -z $(git status -s) ]] | |
| webknossos_windows: | |
| needs: changes | |
| if: | | |
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | |
| ${{ needs.changes.outputs.webknossos == 'true' }} | |
| runs-on: windows-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| group: [1, 2, 3] | |
| fail-fast: false | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "webknossos/uv.lock" | |
| - name: Install proxay | |
| run: npm install -g proxay | |
| - name: Install minio | |
| run: Invoke-WebRequest -Uri "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" -OutFile "minio.exe" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Python tests | |
| timeout-minutes: 30 | |
| shell: bash | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| MULTIPROCESSING_DEFAULT_START_METHOD: spawn | |
| run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | |
| webknossos_macos: | |
| needs: changes | |
| if: | | |
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | |
| ${{ needs.changes.outputs.webknossos == 'true' }} | |
| runs-on: macos-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| group: [1, 2, 3] | |
| fail-fast: false | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "webknossos/uv.lock" | |
| - name: Install proxay | |
| run: npm install -g proxay | |
| - name: Install minio | |
| run: brew install minio/stable/minio | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Python tests | |
| timeout-minutes: 30 | |
| shell: bash | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | |
| coverage_report: | |
| needs: [webknossos_linux] | |
| if: success() | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install coverage.py | |
| run: pip install coverage | |
| - name: Download all coverage reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ~/coverage-files | |
| - name: Generate coverage report | |
| env: | |
| COVERAGE_FILE: "~/coverage-files/result.coverage" | |
| run: | | |
| coverage combine ~/coverage-files/**/*.coverage | |
| coverage xml -i -o ~/coverage-files/result.xml | |
| - name: Report coverage | |
| uses: orgoro/[email protected] | |
| with: | |
| coverageFile: /home/runner/coverage-files/result.xml | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| thresholdAll: 0.8 | |
| thresholdNew: 0.8 | |
| - name: Cleanup temporary files | |
| run: rm -rf ~/coverage-files | |
| webknossos_cli_docker: | |
| needs: | |
| - cluster_tools_slurm | |
| - cluster_tools_multiprocessing | |
| - cluster_tools_kubernetes | |
| - cluster_tools_dask | |
| - webknossos_linux | |
| - webknossos_windows | |
| - webknossos_macos | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Write version file | |
| run: | | |
| pushd webknossos | |
| PKG_VERSION="$(uvx dunamai from git)" | |
| echo "__version__ = '$PKG_VERSION'" > ./webknossos/version.py | |
| sed -i 's/version = "0.0.0"/version = "'"${PKG_VERSION}"'"/g' pyproject.toml | |
| popd | |
| - name: Build docker image | |
| run: docker build -t scalableminds/webknossos-cli:$GITHUB_SHA -f webknossos/Dockerfile . | |
| - name: Smoke test docker | |
| run: | | |
| docker run --rm \ | |
| -v$(pwd)/webknossos/testdata:/webknossos/testdata \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| webknossos convert \ | |
| --jobs 2 \ | |
| --voxel-size 1,1,1 \ | |
| testdata/tiff testoutput/tiff | |
| - name: Login to docker | |
| env: | |
| DOCKER_USER: ${{ secrets.DOCKER_USER }} | |
| DOCKER_PASS: ${{ secrets.DOCKER_PASS }} | |
| run: | | |
| echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin | |
| - name: Push docker images | |
| run: | | |
| docker push scalableminds/webknossos-cli:$GITHUB_SHA | |
| - name: Push docker images (for tag) | |
| if: github.ref_type == 'tag' | |
| run: | | |
| CI_TAG=$(git describe --tags) | |
| docker tag \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| scalableminds/webknossos-cli:$CI_TAG | |
| docker push scalableminds/webknossos-cli:$CI_TAG | |
| docker tag \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| scalableminds/webknossos-cli:latest | |
| docker push scalableminds/webknossos-cli:latest | |
| - name: Push docker images (for branch) | |
| if: github.ref_type == 'branch' | |
| run: | | |
| CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} | |
| NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_} | |
| docker tag \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH | |
| docker push scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH | |
| docs: | |
| needs: | |
| - cluster_tools_slurm | |
| - cluster_tools_multiprocessing | |
| - cluster_tools_kubernetes | |
| - cluster_tools_dask | |
| - webknossos_linux | |
| - webknossos_windows | |
| - webknossos_macos | |
| runs-on: ubuntu-latest | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !github.event.pull_request.head.repo.fork | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: scalableminds/webknossos | |
| path: docs/wk-repo | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Build Docs | |
| run: | | |
| cd docs | |
| ./generate.sh --persist | |
| - name: Push docs (for branch) | |
| if: github.ref_type == 'branch' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: "eu-west-1" | |
| run: | | |
| CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} | |
| NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_} | |
| aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${NORMALIZED_CI_BRANCH} | |
| - name: Push docs (for tag) | |
| if: github.ref_type == 'tag' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: "eu-west-1" | |
| run: | | |
| CI_TAG=$(git describe --tags) | |
| aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${CI_TAG} | |
| - name: Check links (on master) | |
| if: github.ref == 'refs/heads/master' | |
| env: # Or as an environment variable | |
| SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }} | |
| run: | | |
| cd docs | |
| uv run --frozen linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \ | |
| curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \ | |
| "$SLACK_HOOK" | |
| pypi_and_gh_release: | |
| needs: | |
| - cluster_tools_slurm | |
| - cluster_tools_multiprocessing | |
| - cluster_tools_kubernetes | |
| - cluster_tools_dask | |
| - webknossos_linux | |
| - webknossos_windows | |
| - webknossos_macos | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| github.ref_type == 'tag' && | |
| !github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Publish python packages | |
| env: | |
| UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} | |
| run: _tooling/publish.sh | |
| - name: Prepare github release | |
| run: | | |
| VERSION="$(uvx dunamai from git)" | |
| _tooling/changelog_for_version.sh $VERSION > Changelog.md | |
| - name: Publish github release | |
| id: create_release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| tag_name: ${{ github.ref }} | |
| release_name: ${{ github.ref }} | |
| body_path: Changelog.md | |
| draft: false | |
| prerelease: false | |
| complete: | |
| needs: | |
| - cluster_tools_dask | |
| - cluster_tools_kubernetes | |
| - cluster_tools_multiprocessing | |
| - cluster_tools_slurm | |
| - webknossos_linux | |
| - webknossos_windows | |
| - webknossos_macos | |
| - webknossos_cli_docker | |
| - docs | |
| - pypi_and_gh_release | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check failure | |
| if: | | |
| contains(needs.*.result, 'failure') || | |
| contains(needs.*.result, 'cancelled') | |
| run: exit 1 | |
| - name: Success | |
| run: echo Success! |