Merge pull request #121 from crazy-max/hack-split-test-benchmark

ci: split test and benchmark jobs

Author: crazy-max

.github/workflows/ci.yml

@@ -97,6 +97,75 @@ jobs:
       artifact_key: ${{ needs.get-env.outputs.BUILDKIT_ARTIFACT_KEY }}
 
   test:
+    runs-on: ubuntu-24.04
+    needs:
+      - buildkit-binaries
+    env:
+      TEST_FLAGS: -v
+      TEST_IMAGE_BUILD: 0
+      TEST_IMAGE_ID: buildkit-bench
+      TEST_RESULTS_DIR: bin/results
+    steps:
+      -
+        name: Download binaries
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/buildkit-binaries
+          pattern: ${{ env.BUILDKIT_ARTIFACT_KEY }}-*
+          merge-multiple: true
+      -
+        name: Extract binaries
+        run: |
+          mkdir -p ./bin/buildkit-binaries
+          for f in "/tmp/buildkit-binaries"/*.tar.gz; do
+            (set -x ; tar -xzvf "$f" -C ./bin/buildkit-binaries && rm "$f")
+          done
+          tree -nph ./bin/buildkit-binaries
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: ${{ env.SETUP_BUILDX_VERSION }}
+          driver: docker
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Build test image
+        uses: docker/bake-action@v5
+        with:
+          source: "{{defaultContext}}"
+          targets: tests
+          provenance: false
+          set: |
+            *.cache-from=type=registry,ref=${{ env.BUILDKIT_CACHE_REPO }}:tests-base
+            *.contexts.buildkit-binaries=cwd://bin/buildkit-binaries
+            *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Test
+        run: |
+          ./hack/test
+        env:
+          TEST_TYPES: test
+      -
+        name: Result
+        run: |
+          resultPath=./${{ env.TEST_RESULTS_DIR }}/gotestoutput-tests.json
+          mv ./${{ env.TEST_RESULTS_DIR }}/gotestoutput.json $resultPath
+          jq . $resultPath
+      -
+        name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: tests-results
+          path: ${{ env.TEST_RESULTS_DIR }}
+          if-no-files-found: error
+          retention-days: 1
+
+  benchmark:
     runs-on: ubuntu-24.04
     needs:
       - prepare
@@ -110,7 +179,6 @@ jobs:
       TEST_IMAGE_BUILD: 0
       TEST_IMAGE_ID: buildkit-bench
       TEST_RESULTS_DIR: bin/results
-      TEST_GEN: 0
     steps:
       -
         name: Download binaries
@@ -151,10 +219,11 @@ jobs:
         name: Checkout
         uses: actions/checkout@v4
       -
-        name: Test
+        name: Benchmark
         run: |
           ./hack/test
         env:
+          TEST_TYPES: benchmark
           TEST_BENCH_REGEXP: ${{ matrix.test }}$
           TEST_BENCH_RUN: ${{ matrix.count }}
           TEST_BENCH_TIME: ${{ matrix.benchtime }}
@@ -179,6 +248,7 @@ jobs:
     runs-on: ubuntu-24.04
     needs:
       - test
+      - benchmark
     steps:
       -
         name: Download results

README.md

@@ -26,6 +26,12 @@ changes from default branch. You can also specify a commit to test or multiple
 references and tweak the benchmark settings:
 
 ```bash
+# run only tests
+TEST_TYPES=test make test
+
+# run only benchmarks
+TEST_TYPES=benchmark make test
+
 # run a specific benchmark
 TEST_BENCH_REGEXP=/BenchmarkBuildLocal$ make test
 

hack/test

@@ -5,6 +5,7 @@ set -eu -o pipefail
 : "${GITHUB_ACTIONS=}"
 : "${GITHUB_EVENT_NAME=}"
 
+: "${TEST_TYPES=test benchmark}"
 : "${TEST_IMAGE_BUILD=1}"
 : "${TEST_IMAGE_ID=buildkit-bench}"
 : "${TEST_RESULTS_DIR=bin/results}"
@@ -51,6 +52,27 @@ if [ "$GITHUB_ACTIONS" = "true" ]; then
   fi
 fi
 
+gotestArgs="-json -mod=vendor"
+testTypeSet=false
+for testType in $TEST_TYPES; do
+  case "$testType" in
+    test)
+      testTypeSet=true
+      ;;
+    benchmark)
+      gotestArgs="$gotestArgs -bench=${TEST_BENCH_REGEXP:-.} -benchtime=${TEST_BENCH_TIME:-1x} -benchmem"
+      ;;
+    *)
+      echo "Unknown test type: $testType"
+      exit 1
+      ;;
+  esac
+done
+if [ "$testTypeSet" = false ]; then
+  gotestArgs="$gotestArgs -run=^$"
+fi
+gotestArgs="$gotestArgs ${TEST_FLAGS:--v} ${TEST_PKG:-./test/...}"
+
 (
   set -x
   docker run --rm --privileged $dockerConfigMount \
@@ -62,5 +84,5 @@ fi
     -e BUILDKIT_REF_RANDOM=$BUILDKIT_REF_RANDOM \
     -e REGISTRY_MIRROR_DIR=/root/.cache/registry \
     $TEST_IMAGE_ID \
-    sh -c "go test -bench=${TEST_BENCH_REGEXP:-.} -benchtime=${TEST_BENCH_TIME:-1x} -benchmem -json -mod=vendor ${TEST_FLAGS:--v} ${TEST_PKG:-./test/...} | gotestmetrics parse --output $TEST_OUT_DIR/gotestoutput.json"
+    sh -c "go test ${gotestArgs} | gotestmetrics parse --output $TEST_OUT_DIR/gotestoutput.json"
 )

test/build_test.go

@@ -28,7 +28,6 @@ func BenchmarkBuild(b *testing.B) {
 		benchmarkBuildHighParallelization32x,
 		benchmarkBuildHighParallelization64x,
 		benchmarkBuildHighParallelization128x,
-		benchmarkBuildHighParallelization256x,
 		benchmarkBuildFileTransfer,
 	), testutil.WithMirroredImages(mirroredImages))
 }
@@ -117,9 +116,6 @@ func benchmarkBuildHighParallelization64x(b *testing.B, sb testutil.Sandbox) {
 func benchmarkBuildHighParallelization128x(b *testing.B, sb testutil.Sandbox) {
 	benchmarkBuildHighParallelization(b, sb, 128)
 }
-func benchmarkBuildHighParallelization256x(b *testing.B, sb testutil.Sandbox) {
-	benchmarkBuildHighParallelization(b, sb, 256)
-}
 func benchmarkBuildHighParallelization(b *testing.B, sb testutil.Sandbox, n int) {
 	dockerfile := []byte(`
 FROM busybox:latest AS base

testconfig.yml

@@ -60,13 +60,6 @@ runs:
         duration:
           description: Time (s)
           chart: boxplot
-    BenchmarkBuildHighParallelization256x:
-      description: Build high parallelization (256x)
-      count: 4
-      metrics:
-        duration:
-          description: Time (s)
-          chart: boxplot
     BenchmarkBuildFileTransfer:
       description: Build with substantial file transfer
       count: 4