upgrade_downgrade_test_backups_manual_next_release.yml

name: Backups - Manual - Next Release - Upgrade Downgrade Testing
on:
  push:
    branches:
      - "main"
      - "release-[0-9]+.[0-9]"
    tags: '**'
  pull_request:
    branches: '**'

concurrency:
  group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual - Next Release')
  cancel-in-progress: true

permissions: read-all

jobs:

  # This job usually execute in ± 20 minutes
  upgrade_downgrade_test_manual:
    timeout-minutes: 40
    name: Run Upgrade Downgrade Test - Backups - Manual - Next Release
    runs-on: oracle-vm-16cpu-64gb-x86-64

    steps:
    - name: Harden the runner (Audit all outbound calls)
      uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
      with:
        egress-policy: audit

    - name: Skip CI
      run: |
        if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
          echo "skipping CI due to the 'Skip CI' label"
          exit 1
        fi

    # Checkout to this build's commit
    - name: Checkout to commit's code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        fetch-depth: 0
        persist-credentials: 'false'

    - name: Check for changes in relevant files
      if: steps.output-next-release-ref.outputs.next_release_ref != ''
      uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
      id: changes
      with:
        token: ''
        filters: |
          end_to_end:
            - 'test/config.json'
            - 'go/**'
            - 'go/**/*.go'
            - 'test.go'
            - 'Makefile'
            - 'build.env'
            - 'go.sum'
            - 'go.mod'
            - 'proto/*.proto'
            - 'tools/**'
            - 'config/**'
            - 'bootstrap.sh'
            - '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml'
            - 'examples/**'

    - name: Set output with latest release branch
      id: output-next-release-ref
      if: steps.changes.outputs.end_to_end == 'true'
      run: |
        next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}})
        echo $next_release_ref
        echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT

    - name: Tune the OS
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      uses: ./.github/actions/tune-os

    - name: Set up python
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0

    - name: Setup MySQL
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 8
      uses: ./.github/actions/setup-mysql
      with:
        flavor: mysql-8.4

    - name: Setup Percona Apt repository
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      uses: ./.github/actions/setup-percona-repo

    - name: Get base dependencies
      timeout-minutes: 10
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        sudo DEBIAN_FRONTEND="noninteractive" apt-get update

        # Install everything else we need, and configure
        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep
        
        sudo service etcd stop

        sudo percona-release enable tools
        sudo percona-release setup pxb-80
        sudo apt-get update
        sudo apt-get install -y percona-xtrabackup-80

    # Checkout to the next release of Vitess
    - name: Checkout to the other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }})
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
        persist-credentials: 'false'

    - name: Set up Go
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
      with:
        go-version-file: go.mod
        cache: false

    - name: Get dependencies for the next release
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      run: |
        go mod download

    - name: Building next release's binaries
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      run: |
        source build.env
        NOVTADMINBUILD=1 make build
        mkdir -p /tmp/vitess-build-other/
        cp -R bin /tmp/vitess-build-other/
        rm -Rf bin/*

    # Checkout to this build's commit
    - name: Checkout to commit's code
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        persist-credentials: 'false'

    - name: Set up Go
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
      with:
        go-version-file: go.mod
        cache: false

    - name: Get dependencies for this commit
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      run: |
        go mod download

    - name: Run make minimaltools
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      run: |
        make minimaltools

    - name: Building the binaries for this commit
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      run: |
        source build.env
        NOVTADMINBUILD=1 make build
        mkdir -p /tmp/vitess-build-current/
        cp -R bin /tmp/vitess-build-current/

    # We create a sharded Vitess cluster following the local example.
    # We also insert a few rows in our three tables.
    - name: Create the example Vitess cluster with all components using version N
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 10
      run: |
        source build.env ; cd examples/backups
        ./start_cluster.sh

    # Taking a backup
    - name: Take a backup of all the shards
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 5
      run: |
        source build.env ; cd examples/backups
        ./take_backups.sh

    # We insert more data in every table after the backup.
    # When we restore the backup made in the next step, we do not want to see the rows we are about to insert now.
    # The initial number of rows for each table is:
    #     - customer: 5
    #     - product:  2
    #     - corder:   5
    # We shall see the same number of rows after restoring the backup.
    - name: Insert more data after the backup
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        source build.env ; cd examples ; source ./common/env.sh

        echo "insert into customer(email) values('new_user_1@domain.com');" | mysql
        echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
        echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql

    # Stop all the tablets and remove their data
    - name: Stop tablets
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 10
      run: |
        source build.env ; cd examples/backups
        ./stop_tablets.sh

    # We downgrade: we use the version N+1 of vttablet
    - name: Downgrade - Swap binaries, use VTTablet N+1
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        source build.env

        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
        cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
        vttablet --version

    # Starting the tablets again, they will automatically start restoring the last backup.
    - name: Start new tablets and restore
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 10
      run: |
        source build.env ; cd examples/backups
        ./restart_tablets.sh
        # give enough time to the tablets to restore the backup
        sleep 60

    # Count the number of rows in each table to make sure the restoration is successful.
    - name: Assert the number of rows in every table
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        source build.env ; cd examples ; source ./common/env.sh

        echo "select count(sku) from product;" | mysql 2>&1| grep 2
        echo "select count(email) from customer;" | mysql 2>&1| grep 5
        echo "select count(sku) from corder;" | mysql 2>&1| grep 5

    # We insert one more row in every table.
    - name: Insert more rows in the tables
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        source build.env ; cd examples ; source ./common/env.sh

        echo "insert into customer(email) values('new_user_2@domain.com');" | mysql
        echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
        echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql

    # Taking a second backup of the cluster.
    - name: Take a second backup of all the shards
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 10
      run: |
        source build.env ; cd examples/backups
        ./take_backups.sh

    # We upgrade: we swap binaries and use the version N of the tablet.
    - name: Upgrade - Swap binaries, use VTTablet N
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        source build.env

        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
        cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
        vttablet --version

    # Starting the tablets again and restoring the next backup.
    - name: Start new tablets and restore
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      timeout-minutes: 10
      run: |
        source build.env ; cd examples/backups
        ./upgrade_cluster.sh

    # We count the number of rows in every table to check that the restore step was successful.
    - name: Assert the number of rows in every table
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true'
      run: |
        source build.env ; cd examples ; source ./common/env.sh

        echo "select count(sku) from product;" | mysql 2>&1| grep 3
        echo "select count(email) from customer;" | mysql 2>&1| grep 6
        echo "select count(sku) from corder;" | mysql 2>&1| grep 6

    - name: Stop the Vitess cluster
      if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' && !cancelled()
      run: |
        source build.env ; cd examples/local
        ./401_teardown.sh || true