Enable github caching of Bazel's repository cache.

This should substantially reduce network flakes, since most builds will no longer need to download external dependencies.  Because these caches actually get pretty expensive (~300 MB just from 1 openjdk dependency), we've taken some precautions:

1) The caches only get updated on push events (i.e. post-submit jobs)
2) Only a single job will push a new cache for any os/branch/commit combination.  This will be whichever job finishes its bazel run first, and *also* has updates for the cache.  Over time, the cache should expand to the full set of transitive dependencies as the quicker jobs stop pushing updates.
3) A clear_caches workflow has been introduced to reset the caches in case they grow too large.  This will run monthly for now, but can also be run manually.

PiperOrigin-RevId: 508133044

Author: mkruskal-google

.github/actions/bazel-docker/action.yml

@@ -45,6 +45,10 @@ runs:
         credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
         bazel-cache: ${{ inputs.bazel-cache }}
 
+    - name: Hook up repository Cache
+      shell: bash
+      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache='/workspace/${{ env.REPOSITORY_CACHE_PATH }}'" >> $GITHUB_ENV
+
     - name: Validate inputs
       if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
       shell: bash
@@ -63,4 +67,9 @@ runs:
       if: ${{ !inputs.bash }}
       with:
         image: ${{ inputs.image }}
-        command: ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}
+        command: ${{ inputs.bazel }} ${{ env.BAZEL_FLAGS }}
+
+    - name: Save Bazel repository cache
+      # Only allow repository cache updates during post-submits.
+      if: ${{ github.event_name == 'push' }}
+      uses: ./.github/actions/internal/repository-cache-save

.github/actions/bazel/action.yml

@@ -63,11 +63,23 @@ runs:
       run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV
 
     - name: Cache Bazelisk
+      if: ${{ github.event_name == 'push' }}
       uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
       with:
         path: ${{ env.BAZELISK_PATH }}
         key: bazel-${{ runner.os }}-${{ inputs.version }}
 
+    - name: Restore Bazelisk
+      if: ${{ github.event_name != 'push' }}
+      uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+      with:
+        path: ${{ env.BAZELISK_PATH }}
+        key: bazel-${{ runner.os }}-${{ inputs.version }}
+
+    - name: Hook up repository Cache
+      shell: bash
+      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache=$(pwd)/${{ env.REPOSITORY_CACHE_PATH }}" >> $GITHUB_ENV
+
     - name: Validate inputs
       if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
       shell: bash
@@ -90,5 +102,10 @@ runs:
       if: ${{ !inputs.bash }}
       run: >-
         bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
-        ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}
+        ${{ inputs.bazel }} $BAZEL_FLAGS
       shell: bash
+
+    - name: Save Bazel repository cache
+      # Only allow repository cache updates during post-submits.
+      if: ${{ github.event_name == 'push' }}
+      uses: ./.github/actions/internal/repository-cache-save

.github/actions/ccache/action.yml

@@ -25,17 +25,15 @@ runs:
       with:
         path: .ccache
         # Always push to a cache key unique to this commit.
-        key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref, github.sha) }}
+        key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }}
         # Select a cache to restore from with the follow order of preference:
         # 1) The exact same commit we're running over
         # 2) The latest cache from the current ref branch
         # 3) The latest push to the base ref of a pull request
         restore-keys: |
-          ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref, github.sha) }}
-          ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref) }}
+          ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }}
+          ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }}
           ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }}
-          ${{ format('ccache-{0}-refs/heads/{1}', inputs.cache-prefix, github.ref) }}
-          ${{ format('ccache-{0}-refs/heads/{1}', inputs.cache-prefix, github.base_ref) }}
 
     - name: Configure ccache environment variables
       shell: bash

.github/actions/internal/bazel-setup/action.yml

@@ -66,3 +66,8 @@ runs:
       run: |
         echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT
         echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT
+
+    - name: Restore Bazel repository cache
+      uses: ./.github/actions/internal/repository-cache-restore
+      with:
+        bazel-cache: ${{ inputs.bazel-cache }}

.github/actions/internal/repository-cache-restore/action.yml

@@ -0,0 +1,41 @@
+name: Restore Repository Cache
+description: Restore the Bazel repository cache from our github action cache
+inputs:
+  bazel-cache:
+    required: true
+    description: A unique path for the Bazel cache.
+    type: string
+
+# By design, these actions will restore the latest cache for this branch/os,
+# and only save a new version if something has changed.  Initially this will
+# cause a lot of churn, since each test has a slightly different set of
+# repositories to download.  Over time though, since we don't upload no-op
+# changes, this should converge to a stable set of 3 caches per branch.  Every
+# run will update the current cache with a new test's repositories, until there
+# are no unique ones left.
+#
+# This saves asymptotic space, since each one of these can get up to ~500 MB
+# and Github prunes the cache after 10 GB.
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup Bazel repository cache variables
+      shell: bash
+      run: |
+        REPOSITORY_CACHE_BASE=repository-cache-${{ github.base_ref || github.ref_name }}-${{ runner.os }}
+        echo "REPOSITORY_CACHE_BASE=$REPOSITORY_CACHE_BASE" >> $GITHUB_ENV
+        echo "REPOSITORY_CACHE_NAME=$REPOSITORY_CACHE_BASE-${{ inputs.bazel-cache}}-${{ github.sha }}" >> $GITHUB_ENV
+        echo "REPOSITORY_CACHE_PATH=.repository-cache" >> $GITHUB_ENV
+
+    - name: Restore Bazel repository cache
+      id: restore-cache
+      uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+      with:
+        path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }}
+        key: ${{ env.REPOSITORY_CACHE_NAME }}
+        restore-keys: ${{ env.REPOSITORY_CACHE_BASE }}
+
+    - name: Initialize BAZEL environment variable
+      if: ${{ steps.restore-cache.cache-hit }}
+      shell: bash
+      run: echo "REPOSITORY_CACHE_HASH=${{ hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}" >> $GITHUB_ENV

.github/actions/internal/repository-cache-save/action.yml

@@ -0,0 +1,19 @@
+name: Restore Repository Cache
+description: Restore the Bazel repository cache from our github action cache
+
+# Note: this action will only work if repository-cache-restore has already
+# been called.  All bazel actions should specify the repository_cache parameter
+# using REPOSITORY_CACHE_PATH.
+#
+# We intentionally upload to REPOSITORY_CACHE_BASE to prevent a flood of new
+# caches on any change.  Only 1 job per os in each test run will be allowed to
+# update the cache because they're all trying to write to the same location.
+runs:
+  using: 'composite'
+  steps:
+    - name: Save modified Bazel repository cache
+      if: ${{ env.REPOSITORY_CACHE_HASH != hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}
+      uses: actions/cache/save@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+      with:
+        path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }}
+        key: ${{ env.REPOSITORY_CACHE_BASE }}-${{ github.sha }}

.github/workflows/clear_caches.yml

@@ -0,0 +1,29 @@
+name: Clear expensive caches to prevent unbounded growth
+
+on:
+  schedule:
+    # Run every 1st of the month at 10 AM UTC (2 AM PDT)
+    - cron: 0 10 1 * *
+
+  # manual
+  workflow_dispatch:
+
+jobs:
+  bazel-repository-cache:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    name: Clear Bazel repository cache ${{ runner.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+        with:
+          path: ${{ github.workspace }}/${{ steps.output.outputs.repository-cache }}
+          key: repository-cache-${{ github.ref_name }}-${{ runner.os }}-reset-${{ github.sha }}
+
+      - name: Create an empty cache with a single file
+        run: |
+          rm -rf .repository-cache
+          mkdir -p .repository-cache'
+          touch .repository-cache/reset_file