ci.yml

name: ci

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  push:
    branches:
      - 'master'
  pull_request:

env:
  GO_VERSION: 1.22
  SETUP_BUILDX_VERSION: latest
  SETUP_BUILDKIT_IMAGE: moby/buildkit:latest
  BUILDKIT_REPO: moby/buildkit
  BUILDKIT_ARTIFACT_KEY: buildkit-binaries
  BUILDKIT_CACHE_REPO: moby/buildkit-bench-cache
  CANDIDATES_REFS: master
  CANDIDATES_LAST_DAYS: 15
  CANDIDATES_LAST_RELEASES: 8
  WEBSITE_PUBLIC_PATH: /buildkit-bench/

jobs:
  # limitation to using envs in a reusable workflow input
  # https://github.com/actions/runner/issues/2372
  get-env:
    runs-on: ubuntu-24.04
    outputs:
      BUILDKIT_REPO: ${{ env.BUILDKIT_REPO }}
      BUILDKIT_ARTIFACT_KEY: ${{ env.BUILDKIT_ARTIFACT_KEY }}
      CANDIDATES_REFS: ${{ env.CANDIDATES_REFS }}
      CANDIDATES_LAST_DAYS: ${{ env.CANDIDATES_LAST_DAYS }}
      CANDIDATES_LAST_RELEASES: ${{ env.CANDIDATES_LAST_RELEASES }}
    steps:
      - run: "true"

  prepare:
    runs-on: ubuntu-24.04
    outputs:
      includes: ${{ steps.set.outputs.includes }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@v4
      -
        name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      -
        name: Set includes
        id: set
        run: |
          go run -mod=vendor ./cmd/gotestmetrics \
            --debug list --gha-output includes
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
          buildkitd-flags: --debug
      -
        name: Login to Docker Hub
        if: github.event_name != 'pull_request'
        uses: docker/login-action@v3
        with:
          username: ${{ vars.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      -
        name: Build tests base
        uses: docker/bake-action@v5
        with:
          source: "{{defaultContext}}"
          targets: tests-base
          provenance: false
          set: |
            *.cache-from=type=registry,ref=${{ env.BUILDKIT_CACHE_REPO }}:tests-base
            *.cache-to=type=registry,ignore-error=true,mode=max,ref=${{ env.BUILDKIT_CACHE_REPO }}:tests-base

  buildkit-binaries:
    uses: ./.github/workflows/.buildkit-binaries.yml
    needs:
      - get-env
    secrets: inherit
    with:
      repo: ${{ needs.get-env.outputs.BUILDKIT_REPO }}
      refs: ${{ needs.get-env.outputs.CANDIDATES_REFS }}
      last_days: ${{ needs.get-env.outputs.CANDIDATES_LAST_DAYS }}
      last_releases: ${{ needs.get-env.outputs.CANDIDATES_LAST_RELEASES }}
      artifact_key: ${{ needs.get-env.outputs.BUILDKIT_ARTIFACT_KEY }}

  test:
    runs-on: ubuntu-24.04
    needs:
      - prepare
      - buildkit-binaries
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.prepare.outputs.includes) }}
    env:
      TEST_FLAGS: -v --timeout=5h
      TEST_IMAGE_BUILD: 0
      TEST_IMAGE_ID: buildkit-bench
      TEST_RESULTS_DIR: bin/results
      TEST_GEN: 0
    steps:
      -
        name: Download binaries
        uses: actions/download-artifact@v4
        with:
          path: /tmp/buildkit-binaries
          pattern: ${{ env.BUILDKIT_ARTIFACT_KEY }}-*
          merge-multiple: true
      -
        name: Extract binaries
        run: |
          mkdir -p ./bin/buildkit-binaries
          for f in "/tmp/buildkit-binaries"/*.tar.gz; do
            (set -x ; tar -xzvf "$f" -C ./bin/buildkit-binaries && rm "$f")
          done
          tree -nph ./bin/buildkit-binaries
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver: docker
      -
        name: Build test image
        uses: docker/bake-action@v5
        with:
          source: "{{defaultContext}}"
          targets: tests
          provenance: false
          set: |
            *.cache-from=type=registry,ref=${{ env.BUILDKIT_CACHE_REPO }}:tests-base
            *.contexts.buildkit-binaries=cwd://bin/buildkit-binaries
            *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
      -
        name: Checkout
        uses: actions/checkout@v4
      -
        name: Test
        run: |
          ./hack/test
        env:
          TEST_BENCH_REGEXP: ${{ matrix.test }}$
          TEST_BENCH_RUN: ${{ matrix.count }}
          TEST_BENCH_TIME: ${{ matrix.benchtime }}
      -
        name: Result
        run: |
          resultName=$(echo ${{ matrix.test }} | sed 's/\//-/g')
          resultPath=./${{ env.TEST_RESULTS_DIR }}/gotestoutput-$resultName.json
          echo "TEST_RESULT_NAME=$resultName" >> $GITHUB_ENV
          echo "TEST_RESULT_PATH=$resultPath" >> $GITHUB_ENV
          mv ./${{ env.TEST_RESULTS_DIR }}/gotestoutput.json $resultPath
          jq . $resultPath
      -
        name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-${{ env.TEST_RESULT_NAME }}
          path: ${{ env.TEST_RESULT_PATH }}
          if-no-files-found: error
          retention-days: 1

  report:
    runs-on: ubuntu-24.04
    needs:
      - test
    steps:
      -
        name: Download results
        uses: actions/download-artifact@v4
        with:
          path: /tmp/buildkit-bench
          pattern: bench-results-*
          merge-multiple: true
      -
        name: Download candidates
        uses: actions/download-artifact@v4
        with:
          name: candidates
          path: /tmp/buildkit-bench
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
          buildkitd-flags: --debug
      -
        name: Checkout
        uses: actions/checkout@v4
      -
        name: Generate HTML report
        uses: docker/bake-action@v5
        with:
          source: "{{defaultContext}}"
          targets: tests-gen
          provenance: false
          set: |
            *.cache-from=type=registry,ref=${{ env.BUILDKIT_CACHE_REPO }}:tests-base
            *.contexts.tests-results=cwd:///tmp/buildkit-bench
            *.output=./bin/report
        env:
          BAKE_ALLOW_REMOTE_FS_ACCESS: 1
      -
        name: Include additional files to report directory
        run: |
          cp /tmp/buildkit-bench/* ./testconfig.yml ./bin/report/
          cat <<EOF > ./bin/report/metadata.txt
          repo=${{ github.repository }}
          commit=${{ github.sha }}
          EOF
      -
        name: Upload report
        uses: actions/upload-artifact@v4
        with:
          name: report
          path: ./bin/report
          if-no-files-found: error

  publish:
    runs-on: ubuntu-24.04
    needs:
      - report
    steps:
      -
        name: Checkout
        uses: actions/checkout@v4
      -
        name: Checkout gh-pages
        uses: actions/checkout@v4
        with:
          ref: gh-pages
          path: bin/gh-pages
      -
        name: Download report
        uses: actions/download-artifact@v4
        with:
          name: report
          path: /tmp/buildkit-bench-report
      -
        name: Move reports
        run: |
          reportDir=$(date +'%Y%m%d-%H%M%S')
          mkdir -p ./website/public/result/$reportDir
          mv /tmp/buildkit-bench-report/* ./website/public/result/$reportDir/
          if [ -d ./bin/gh-pages/result ]; then
            mv ./bin/gh-pages/result/* ./website/public/result/
          fi
      -
        name: Build website
        uses: docker/bake-action@v5
        with:
          targets: website
          provenance: false
      -
        name: Publish
        uses: crazy-max/ghaction-github-pages@v4
        with:
          target_branch: gh-pages
          build_dir: ./bin/website
          jekyll: false
          dry_run: ${{ github.event_name == 'pull_request' }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}